diff --git a/.github/workflows/llgo.yml b/.github/workflows/llgo.yml index 7b7c98a0cc..a557984c9c 100644 --- a/.github/workflows/llgo.yml +++ b/.github/workflows/llgo.yml @@ -315,14 +315,23 @@ jobs: (cd ./_demo/c/cargs && llgo run .) - name: Test SIMD behavior with Go and LLGo + id: native_simd if: matrix.windows_arch != '386' shell: bash env: GOEXPERIMENT: simd + LLGO_SIMD_ARTIFACT_DIR: ${{ runner.temp }}/llgo-simd-matrix run: | go test -count=1 -timeout=2m ./test/simd/... - llgo test -O0 -count=1 -timeout=2m ./test/simd/... - llgo test -O2 -count=1 -timeout=2m ./test/simd/... + bash dev/test_native_simd.sh + + - name: Upload failed native SIMD executables + if: failure() && steps.native_simd.outcome == 'failure' + uses: actions/upload-artifact@v7 + with: + name: native-simd-${{ matrix.os }}-${{ matrix.windows_abi || 'native' }}-${{ matrix.windows_arch || 'host' }} + path: ${{ runner.temp }}/llgo-simd-matrix + retention-days: 3 - name: Test demos run: bash .github/workflows/test_demo.sh diff --git a/cl/compile.go b/cl/compile.go index e6e18c6885..37fa77bf5a 100644 --- a/cl/compile.go +++ b/cl/compile.go @@ -674,6 +674,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun fn = pkg.NewFuncEx(name, sig, background, false, p.needsLinkOnce(f)) } } + p.markSIMDFMV(fn, f) if p.prog.Target().GOARCH == "wasm" { if decl, ok := f.Syntax().(*ast.FuncDecl); ok { fullName := declarationFuncName(llssa.PathOf(pkgTypes), decl, f.Object(), f.Signature.Recv()) @@ -692,6 +693,9 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun if noInlineDirective || runtimeStackNoInline || pcLineNoInline || usesRecover { fn.DisableTailCalls() } + if !disableInline && !noInlineDirective && (runtimeStackNoInline || pcLineNoInline || usesRecover) { + fn.AllowSIMDDispatcherInlining() + } p.funcs[f] = fn isCgo := isCgoExternSymbol(f) if (len(f.Blocks) != 0 || simdDecl) && p.prog.FuncInfoMetadataEnabled() { diff --git a/cl/instr.go b/cl/instr.go index eb75363efd..7926b7e14f 100644 --- a/cl/instr.go +++ b/cl/instr.go @@ -735,6 +735,9 @@ func (p *context) funcOf(fn *ssa.Function) (aFn llssa.Function, pyFn llssa.PyObj aFn.Inline(llssa.NoInline) } } + if ftype == goFunc { + p.markSIMDFMV(aFn, fn) + } } return } diff --git a/cl/simd_fmv.go b/cl/simd_fmv.go new file mode 100644 index 0000000000..a86b9b8ecd --- /dev/null +++ b/cl/simd_fmv.go @@ -0,0 +1,54 @@ +package cl + +import ( + "go/types" + + "github.com/xgo-dev/llgo/internal/llvmfmv" + llssa "github.com/xgo-dev/llgo/ssa" + "golang.org/x/tools/go/ssa" +) + +func simdCPUQuery(fn *ssa.Function) string { + obj, ok := fn.Object().(*types.Func) + if !ok || obj.Pkg() == nil || obj.Pkg().Path() != "simd/archsimd" || obj.Name() != "AVX2" || + !types.Identical(obj.Type(), fn.Signature) { + return "" + } + sig := fn.Signature + if sig.Recv() == nil || sig.Params().Len() != 0 || sig.Results().Len() != 1 || + !types.Identical(sig.Results().At(0).Type(), types.Typ[types.Bool]) { + return "" + } + recv, ok := sig.Recv().Type().(*types.Named) + if !ok || recv.Obj().Pkg() != obj.Pkg() || recv.Obj().Name() != "X86Features" { + return "" + } + return llvmfmv.AVX2Query +} + +func (p *context) markSIMDFMV(fn llssa.Function, source *ssa.Function) { + if fn == nil || p.prog.Target().GOARCH != "amd64" { + return + } + // Bodyless assembly/linkname declarations do not promise a compiler-emitted + // entry. Official SIMD intrinsic declarations have LLGo-generated bodies. + _, intrinsic := lookupSIMD(source, "amd64") + available := len(source.Blocks) != 0 || intrinsic + hasVector := false + sig := source.Signature + check := func(t types.Type) { + if _, ok := llssa.SIMDVectorShape(t); ok { + hasVector = true + } + } + if sig.Recv() != nil { + check(sig.Recv().Type()) + } + for i := 0; i < sig.Params().Len(); i++ { + check(sig.Params().At(i).Type()) + } + for i := 0; i < sig.Results().Len(); i++ { + check(sig.Results().At(i).Type()) + } + fn.MarkSIMDFMV(available && hasVector, simdCPUQuery(source)) +} diff --git a/cl/simd_test.go b/cl/simd_test.go index 4fe1221452..60ee459825 100644 --- a/cl/simd_test.go +++ b/cl/simd_test.go @@ -13,6 +13,39 @@ import ( "golang.org/x/tools/go/ssa/ssautil" ) +func TestSIMDFMVQueryIdentity(t *testing.T) { + for _, tc := range []struct { + name, path, recv, signature string + want string + }{ + {"official", "simd/archsimd", "X86Features", "AVX2() bool", "x86.avx2"}, + {"other package", "example/archsimd", "X86Features", "AVX2() bool", ""}, + {"other receiver", "simd/archsimd", "OtherFeatures", "AVX2() bool", ""}, + {"pointer receiver", "simd/archsimd", "*X86Features", "AVX2() bool", ""}, + {"other query", "simd/archsimd", "X86Features", "AVX() bool", ""}, + {"argument", "simd/archsimd", "X86Features", "AVX2(int) bool", ""}, + {"result", "simd/archsimd", "X86Features", "AVX2() int", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + fs := token.NewFileSet() + file, err := parser.ParseFile(fs, "cpu.go", "package archsimd; type X86Features struct{}; type OtherFeatures struct{}; func ("+tc.recv+") "+tc.signature, 0) + if err != nil { + t.Fatal(err) + } + pkg, err := new(types.Config).Check(tc.path, fs, []*ast.File{file}, nil) + if err != nil { + t.Fatal(err) + } + prog := ssa.NewProgram(fs, 0) + prog.CreatePackage(pkg, nil, nil, true) + named := pkg.Scope().Lookup(strings.TrimPrefix(tc.recv, "*")).Type().(*types.Named) + if got := simdCPUQuery(prog.FuncValue(named.Method(0))); got != tc.want { + t.Fatalf("query = %q, want %q", got, tc.want) + } + }) + } +} + func TestSIMDOperationIdentity(t *testing.T) { const source = `package archsimd type v128 struct { _ [0]func() } diff --git a/dev/test_native_simd.sh b/dev/test_native_simd.sh new file mode 100755 index 0000000000..255c1b02db --- /dev/null +++ b/dev/test_native_simd.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +: "${LLGO:=llgo}" +export GOEXPERIMENT=simd + +if [[ "$(go env GOARCH)" != amd64 ]]; then + "$LLGO" test -O0 -count=1 -timeout=2m ./test/simd/... + "$LLGO" test -O2 -count=1 -timeout=2m ./test/simd/... + exit +fi + +export GOAMD64=v1 +if [[ -n "${LLGO_SIMD_ARTIFACT_DIR:-}" ]]; then + output=$LLGO_SIMD_ARTIFACT_DIR + mkdir -p "$output" +else + output=$(mktemp -d) + trap 'rm -rf "$output"' EXIT +fi + +for profile in O0-off O2-off O2-thin O2-full; do + opt=${profile%%-*} + lto=${profile#*-} + flags=("-$opt") + if [[ "$lto" != off ]]; then + flags+=("-lto=$lto") + fi + echo "SIMD: amd64 $profile" + binary="$output/simd-$profile.exe" + "$LLGO" test "${flags[@]}" -c -o "$binary" ./test/simd + "$binary" -test.v -test.count=1 -test.timeout=2m + for debug in cpu.avx2=off cpu.avx=off cpu.fma=off cpu.all=off; do + echo "SIMD FMV: $profile GODEBUG=$debug" + GODEBUG="$debug" "$binary" -test.v -test.count=1 -test.timeout=2m -test.run='^TestSIMDFMV' + done +done diff --git a/go.mod b/go.mod index 8c1b92ebe2..d1009d36fc 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/mattn/go-tty v0.0.8 github.com/qiniu/x v1.19.2 github.com/xgo-dev/llgo/runtime v0.0.0-00010101000000-000000000000 - github.com/xgo-dev/llvm v0.10.0 + github.com/xgo-dev/llvm v0.10.1 github.com/xgo-dev/plan9asm v0.6.2 go.bug.st/serial v1.6.4 go.yaml.in/yaml/v3 v3.0.5 diff --git a/go.sum b/go.sum index 6eae56a476..ef72438e1c 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/xgo-dev/llvm v0.10.0 h1:o27kypAGI7DrLFckrsUVjWChXqHUBDAFtZbwMiDFs50= github.com/xgo-dev/llvm v0.10.0/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= +github.com/xgo-dev/llvm v0.10.1 h1:GZeoapgSKC1Qgfy+xQmlIAKg0Suvrpg4U3Y3xticjqw= +github.com/xgo-dev/llvm v0.10.1/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M= github.com/xgo-dev/plan9asm v0.6.2 h1:9mV7u9stDT2V3wZP+ywRP/wbWmtQwf5bb6zym5Unyww= github.com/xgo-dev/plan9asm v0.6.2/go.mod h1:UTpcZz3aQXSWYruhfzOi8aK4QZTEFPIvB61YW6jHT4M= go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A= diff --git a/internal/build/build.go b/internal/build/build.go index ea2ddf72ae..81efc6dea0 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -56,6 +56,7 @@ import ( "github.com/xgo-dev/llgo/internal/goarch" "github.com/xgo-dev/llgo/internal/goembed" "github.com/xgo-dev/llgo/internal/header" + "github.com/xgo-dev/llgo/internal/llvmfmv" "github.com/xgo-dev/llgo/internal/lto" "github.com/xgo-dev/llgo/internal/meta" "github.com/xgo-dev/llgo/internal/mockable" @@ -3285,6 +3286,7 @@ func lowerMainCExportModule(ctx *context, pkg llssa.Package, exports []cExport) if !lowerMainCExportAggregates(ctx.prog, mod, exports) { return false, nil } + ctx.cTransformer.LowerWindowsVectorParams(mod) ctx.cTransformer.TransformModule(pkg.Path(), mod) if ctx.buildConf.Goarch != "wasm" { return true, nil @@ -3330,6 +3332,10 @@ func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbos ret := aPkg.LPkg ctx.cTransformer.SetSkipFuncs(cabiSkipFuncsForPlan9Asm(ctx, pkgPath, ret.Module())) + ctx.cTransformer.LowerWindowsVectorParams(ret.Module()) + if err := llvmfmv.Run(ret.Module(), ctx.buildConf.GOAMD64); err != nil { + return fmt.Errorf("%s: %w", pkgPath, err) + } lowerLargeAggregates(ctx.prog, ret.Module()) ctx.cTransformer.TransformModule(ret.Path(), ret.Module()) ctx.cTransformer.SetSkipFuncs(nil) diff --git a/internal/build/large_roots_test.go b/internal/build/large_roots_test.go index 46590ae2a6..402196509d 100644 --- a/internal/build/large_roots_test.go +++ b/internal/build/large_roots_test.go @@ -1,6 +1,7 @@ package build import ( + "go/token" "go/types" "strings" "testing" @@ -10,6 +11,44 @@ import ( "github.com/xgo-dev/llvm" ) +func TestLowerMainCExportModuleWindowsVectors(t *testing.T) { + llvm.InitializeAllTargets() + llvm.InitializeAllTargetMCs() + llvm.InitializeAllTargetInfos() + target := &llssa.Target{GOOS: "windows", GOARCH: "amd64"} + prog := llssa.NewProgram(target) + defer prog.Dispose() + pkg := prog.NewPackage("exportprobe", "exportprobe") + rawPkg := types.NewPackage("simd/archsimd", "archsimd") + marker := types.NewNamed(types.NewTypeName(token.NoPos, rawPkg, "v128", nil), types.NewStruct(nil, nil), nil) + vector := types.NewNamed(types.NewTypeName(token.NoPos, rawPkg, "Float32x4", nil), types.NewStruct([]*types.Var{ + types.NewField(token.NoPos, rawPkg, "tag", marker, false), + types.NewField(token.NoPos, rawPkg, "vals", types.NewArray(types.Typ[types.Float32], 4), false), + }, nil), nil) + exports := []cExport{{goName: "main.Exported", cName: "Exported", sig: newSignature([]types.Type{vector}, []types.Type{vector})}} + defineCExportWrappers(pkg, exports, nil) + ctx := &context{ + prog: prog, buildConf: &Config{Goos: "windows", Goarch: "amd64"}, + cTransformer: cabi.NewTransformer(prog, target.Spec().Triple, "", true), + } + if changed, err := lowerMainCExportModule(ctx, pkg, exports); err != nil || !changed { + t.Fatalf("C export lowering = (%t, %v)", changed, err) + } + mod := pkg.Module() + for _, name := range []string{"Exported", "main.Exported"} { + fn := mod.NamedFunction(name) + if fn.Param(0).Type().TypeKind() != llvm.PointerTypeKind || fn.GlobalValueType().ReturnType().TypeKind() != llvm.VectorTypeKind { + t.Fatalf("Win64 export must use an indirect vector parameter and direct vector return:\n%s", fn.String()) + } + } + if !strings.Contains(mod.NamedFunction("Exported").String(), "@main.Exported(ptr ") { + t.Fatalf("export call does not match the lowered implementation:\n%s", mod.String()) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } +} + func TestLowerLargeAggregatesGCPolicy(t *testing.T) { llvm.InitializeAllTargets() llvm.InitializeAllTargetMCs() diff --git a/internal/build/simd_fmv_coff_test.go b/internal/build/simd_fmv_coff_test.go new file mode 100644 index 0000000000..431de3735c --- /dev/null +++ b/internal/build/simd_fmv_coff_test.go @@ -0,0 +1,115 @@ +//go:build !llgo + +package build + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/xgo-dev/llvm" +) + +// Compile the same generic instantiation in two packages and link the actual +// COFF objects. LLVM linkage alone does not deduplicate definitions on Windows. +func TestSIMDFMVWindowsGenericLink(t *testing.T) { + linker, err := exec.LookPath("lld-link") + if err != nil { + t.Fatal("lld-link is required for the Windows FMV link regression") + } + dir := t.TempDir() + for name, source := range map[string]string{ + "go.mod": "module fmvcomdat\n\ngo 1.27\n", + "shared/shared.go": `package shared +import "simd/archsimd" +//go:noinline +func Echo[T ~int](x archsimd.Float32x4, n T) archsimd.Float32x4 { + if archsimd.X86.AVX2() { return x.Add(x) } + return x +}`, + "a/a.go": `package a +import "simd/archsimd" +import "fmvcomdat/shared" +//go:noinline +func A(x archsimd.Float32x4) archsimd.Float32x4 { return shared.Echo(x, 1) }`, + "b/b.go": `package b +import "simd/archsimd" +import "fmvcomdat/shared" +//go:noinline +func B(x archsimd.Float32x4) archsimd.Float32x4 { return shared.Echo(x, 1) }`, + } { + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + for _, level := range []string{"O0", "O2"} { + t.Run(level, func(t *testing.T) { + outDir := t.TempDir() + var objects []string + for _, name := range []string{"a", "b"} { + conf := NewDefaultConf(ModeGen) + conf.Goos, conf.Goarch, conf.GOAMD64, conf.GOEXPERIMENT = "windows", "amd64", "v1", "simd" + pkgs, err := Build(Invocation{Args: []string{"./" + name}, Config: conf, Dir: dir}) + if err != nil { + t.Fatal(err) + } + prog := pkgs[0].LPkg.Prog + defer prog.Dispose() + mod := pkgs[0].LPkg.Module() + variant := mod.NamedFunction("fmvcomdat/shared.Echo[int].__llgo_fmv_avx2") + if variant.IsNil() || variant.IsDeclaration() { + t.Fatal("generic AVX2 definition missing from package", name) + } + writeObject := func(name string, mod llvm.Module) { + t.Helper() + mod.SetTarget(prog.Target().Spec().Triple) + mod.SetDataLayout(prog.DataLayout()) + if level == "O2" { + opts := llvm.NewPassBuilderOptions() + defer opts.Dispose() + if err := mod.RunPasses("default", prog.TargetMachine(), opts); err != nil { + t.Fatal(err) + } + } + buf, err := prog.TargetMachine().EmitToMemoryBuffer(mod, llvm.ObjectFile) + if err != nil { + t.Fatal(err) + } + defer buf.Dispose() + path := filepath.Join(outDir, name+".obj") + if err := os.WriteFile(path, buf.Bytes(), 0600); err != nil { + t.Fatal(err) + } + objects = append(objects, path) + } + writeObject(name, mod) + if name == "a" { + // Supply the fixture's runtime dependencies without linking the + // whole runtime. Unresolved symbols remain fatal to this test. + ctx := mod.Context() + support := ctx.NewModule("support") + defer support.Dispose() + mask := llvm.AddGlobal(support, ctx.Int64Type(), "github.com/xgo-dev/llgo/runtime/internal/runtime.CPUFeatures") + mask.SetInitializer(llvm.ConstNull(ctx.Int64Type())) + b := ctx.NewBuilder() + defer b.Dispose() + for _, init := range []string{"simd/archsimd.init", "fmvcomdat/shared.init"} { + fn := llvm.AddFunction(support, init, llvm.FunctionType(ctx.VoidType(), nil, false)) + b.SetInsertPointAtEnd(ctx.AddBasicBlock(fn, "entry")) + b.CreateRetVoid() + } + writeObject("support", support) + } + } + args := []string{"/dll", "/noentry", "/nodefaultlib", "/opt:noref", "/out:" + filepath.Join(outDir, "generic.dll")} + if out, err := exec.Command(linker, append(args, objects...)...).CombinedOutput(); err != nil { + t.Fatalf("link generic SIMD instantiations: %v\n%s", err, out) + } + }) + } +} diff --git a/internal/build/simd_fmv_test.go b/internal/build/simd_fmv_test.go new file mode 100644 index 0000000000..2b7724049b --- /dev/null +++ b/internal/build/simd_fmv_test.go @@ -0,0 +1,147 @@ +//go:build !llgo + +package build + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +const simdFMVSource = `package main +import "simd/archsimd" +import "fmvtest/kernel" +import _ "unsafe" + +//go:noinline +func local(x, y archsimd.Float32x4) archsimd.Float32x4 { return x.Add(y).Mul(x) } + +//go:linkname assembly fmv.externalAssembly +func assembly(x archsimd.Float32x4) archsimd.Float32x4 + +//go:noinline +func guarded(x, y archsimd.Float32x4) archsimd.Float32x4 { + if archsimd.X86.AVX2() { return assembly(local(x, kernel.Compute(x, y))) } + return x.Sub(y) +} +func main() {} +` + +func simdFMVTestDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := os.Mkdir(filepath.Join(dir, "kernel"), 0700); err != nil { + t.Fatal(err) + } + for name, source := range map[string]string{ + "go.mod": "module fmvtest\n\ngo 1.27\n", + "main.go": simdFMVSource, + "kernel/kernel.go": `package kernel +import "simd/archsimd" +//go:noinline +func Compute(x, y archsimd.Float32x4) archsimd.Float32x4 { return x.Add(y) } +`, + } { + if err := os.WriteFile(filepath.Join(dir, name), []byte(source), 0600); err != nil { + t.Fatal(err) + } + } + return dir +} + +func TestSIMDFMVLLVM(t *testing.T) { + for _, goos := range []string{"linux", "darwin", "windows"} { + for _, level := range []string{"O0", "O2"} { + t.Run(goos+"/v1/"+level, func(t *testing.T) { + testSIMDFMVLLVM(t, goos, "v1", level) + }) + } + } + t.Run("linux/v3/O2", func(t *testing.T) { + testSIMDFMVLLVM(t, "linux", "v3", "O2") + }) +} + +func testSIMDFMVLLVM(t *testing.T, goos, baseline, level string) { + conf := NewDefaultConf(ModeGen) + conf.Goos, conf.Goarch, conf.GOAMD64, conf.GOEXPERIMENT = goos, "amd64", baseline, "simd" + conf.LinkOptions.DWARF = DWARFPreserve + dir := simdFMVTestDir(t) + pkgs, err := Build(Invocation{Args: []string{"."}, Config: conf, Dir: dir}) + if err != nil { + t.Fatal(err) + } + defer pkgs[0].LPkg.Prog.Dispose() + mod := pkgs[0].LPkg.Module() + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + guard := mod.NamedFunction("main.guarded").String() + fast := guard + suffix := "" + if baseline == "v3" || baseline == "v4" { + if strings.Contains(mod.String(), "__llgo_fmv") { + t.Fatalf("baseline feature generated redundant versions:\n%s", mod.String()) + } + } else { + suffix = ".__llgo_fmv_avx2" + if !strings.Contains(mod.String(), `linkageName: "main.guarded.__llgo_fmv_avx2"`) { + t.Fatal("specialized debug subprogram lost its linker identity") + } + baselineBody := mod.NamedFunction("main.guarded.__llgo_fmv_baseline").String() + if goos == "windows" { + for _, name := range []string{"main.guarded", "main.guarded.__llgo_fmv_resolve"} { + fn := mod.NamedFunction(name) + if fn.Param(0).Type().TypeKind() != llvm.PointerTypeKind || strings.Contains(fn.String(), "alloca") || strings.Contains(fn.String(), "load <4 x float>") { + t.Fatalf("Win64 tail transfer must forward caller-owned vector storage:\n%s", fn.String()) + } + } + } + fast = mod.NamedFunction("main.guarded" + suffix).String() + if !strings.Contains(guard, "musttail call") || !strings.Contains(guard, "load atomic ptr") || + strings.Contains(guard, "X86Features.AVX2") || strings.Contains(baselineBody, "X86Features.AVX2") || !strings.Contains(baselineBody, "fsub") { + t.Fatalf("invalid dispatch or baseline:\n%s\n%s", guard, baselineBody) + } + } + if strings.Contains(fast, "X86Features.AVX2") || strings.Contains(fast, "fsub") || + !strings.Contains(fast, "main.local"+suffix) || !strings.Contains(fast, "fmvtest/kernel.Compute"+suffix) { + t.Fatalf("invalid specialization:\n%s", fast) + } + if !strings.Contains(fast, "@fmv.externalAssembly(") || !mod.NamedFunction("fmv.externalAssembly.__llgo_fmv_avx2").IsNil() { + t.Fatal("bodyless assembly declaration must keep its baseline entry") + } + kernels, err := Build(Invocation{Args: []string{"./kernel"}, Config: conf, Dir: dir}) + if err != nil { + t.Fatal(err) + } + defer kernels[0].LPkg.Prog.Dispose() + kernel := kernels[0].LPkg.Module().NamedFunction("fmvtest/kernel.Compute" + suffix) + if kernel.IsNil() || kernel.BasicBlocksCount() == 0 { + t.Fatal("cross-package specialized definition missing") + } + prog := pkgs[0].LPkg.Prog + mod.SetDataLayout(prog.DataLayout()) + mod.SetTarget(prog.Target().Spec().Triple) + if level == "O2" { + opts := llvm.NewPassBuilderOptions() + defer opts.Dispose() + opts.SetVerifyEach(true) + if err := mod.RunPasses("default", prog.TargetMachine(), opts); err != nil { + t.Fatal(err) + } + } + asm, err := prog.TargetMachine().EmitToMemoryBuffer(mod, llvm.AssemblyFile) + if err != nil { + t.Fatal(err) + } + defer asm.Dispose() + if goos == "linux" && baseline == "v1" { + assertSIMDBaselineAssembly(t, string(asm.Bytes())) + } + if !strings.Contains(string(asm.Bytes()), "vaddps") { + t.Fatal("AVX2 version did not select AVX instructions") + } +} diff --git a/internal/build/simd_test.go b/internal/build/simd_test.go index 094de5fc34..f1cd66c77b 100644 --- a/internal/build/simd_test.go +++ b/internal/build/simd_test.go @@ -218,8 +218,8 @@ func TestSIMD128LLVM(t *testing.T) { if !strings.Contains(string(asm.Bytes()), want) { t.Fatalf("missing %s in assembly", want) } - if target.arch == "amd64" && regexp.MustCompile(`(?m)^\s+v[a-z][a-z0-9]*\s`).Match(asm.Bytes()) { - t.Fatal("GOAMD64=v1 emitted an AVX instruction") + if target.arch == "amd64" { + assertSIMDBaselineAssembly(t, string(asm.Bytes())) } if target.arch != "amd64" { wantLookup := map[string]string{"arm64": "tbl", "wasm": "i8x16.swizzle"}[target.arch] @@ -235,6 +235,22 @@ func TestSIMD128LLVM(t *testing.T) { } } +// Linux assembly marks each function explicitly. AVX is legal only inside the +// guarded variants; check every baseline body, including unguarded helpers. +func assertSIMDBaselineAssembly(t *testing.T, asm string) { + t.Helper() + bodies := regexp.MustCompile(`(?ms)^\s*\.type\s+([^\n]+),@function\n(.*?)^\s*\.size\s+[^\n]+`).FindAllStringSubmatch(asm, -1) + if len(bodies) == 0 { + t.Fatal("no assembly functions found") + } + avx := regexp.MustCompile(`(?m)^\s+v[a-z][a-z0-9]*\s`) + for _, body := range bodies { + if !strings.Contains(body[1], ".__llgo_fmv_avx2") && avx.MatchString(body[2]) { + t.Fatalf("GOAMD64=v1 baseline %s emitted AVX:\n%s", body[1], body[2]) + } + } +} + func TestSIMDIntrinsicDefinitions(t *testing.T) { dir := simdTestDir(t) for _, target := range []struct{ os, arch string }{{"windows", "amd64"}, {"windows", "arm64"}, {"linux", "amd64"}, {"linux", "arm64"}, {"wasip1", "wasm"}} { diff --git a/internal/cabi/cabi.go b/internal/cabi/cabi.go index 020e994c5b..bb32f9c353 100644 --- a/internal/cabi/cabi.go +++ b/internal/cabi/cabi.go @@ -148,12 +148,16 @@ type CallInstr struct { } func (p *Transformer) TransformModule(path string, m llvm.Module) { + p.transformModule(m, nil) +} + +func (p *Transformer) transformModule(m llvm.Module, acceptCallConv func(llvm.CallConv) bool) { ctx := m.Context() var fns []llvm.Value var callInstrs []CallInstr fn := m.FirstFunction() for !fn.IsNil() { - if !p.shouldSkipFunc(fn.Name()) && p.isWrapFunctionType(ctx, fn.GlobalValueType()) { + if !p.shouldSkipFunc(fn.Name()) && (acceptCallConv == nil || acceptCallConv(fn.FunctionCallConv())) && p.isWrapFunctionType(ctx, fn.GlobalValueType()) { fns = append(fns, fn) } bb := fn.FirstBasicBlock() @@ -161,7 +165,7 @@ func (p *Transformer) TransformModule(path string, m llvm.Module) { instr := bb.FirstInstruction() for !instr.IsNil() { if call := instr.IsACallInst(); !call.IsNil() { - if p.shouldSkipCall(call) { + if p.shouldSkipCall(call) || (acceptCallConv != nil && !acceptCallConv(call.InstructionCallConv())) { instr = llvm.NextInstruction(instr) continue } @@ -778,6 +782,7 @@ func (p *Transformer) transformCallInstr(m llvm.Module, ctx llvm.Context, call l } updateCallAttr := func(replacement llvm.Value) { + replacement.InstructionSetDebugLoc(call.InstructionDebugLoc()) replacement.SetInstructionCallConv(call.InstructionCallConv()) for i, list := range attrs { for _, attr := range list { diff --git a/internal/cabi/windows_vector.go b/internal/cabi/windows_vector.go new file mode 100644 index 0000000000..788a58d9f5 --- /dev/null +++ b/internal/cabi/windows_vector.go @@ -0,0 +1,40 @@ +package cabi + +import "github.com/xgo-dev/llvm" + +// LowerWindowsVectorParams exposes the native Win64 indirect vector-parameter +// ABI before FMV creates musttail forwarding functions. Otherwise LLVM's late +// ABI lowering copies a vector into the forwarding function's own stack frame, +// then passes a dangling pointer when the tail jump releases that frame. +// Vector results still use XMM0. This does not lower aggregate parameters or +// zero-sized CPU-query receivers, which FMV must inspect in their source form. +func (p *Transformer) LowerWindowsVectorParams(m llvm.Module) { + if _, ok := p.sys.(*TypeInfoWindowsAmd64); !ok { + return + } + narrow := *p + narrow.sys = &windowsVectorParams{p} + narrow.transformModule(m, func(cc llvm.CallConv) bool { + // 79 is LLVM's explicit Win64 calling convention. + return cc == llvm.CCallConv || cc == llvm.CallConv(79) + }) +} + +type windowsVectorParams struct{ *Transformer } + +func (*windowsVectorParams) SupportByVal() bool { return false } +func (*windowsVectorParams) SkipEmptyParams() bool { return false } + +func (p *windowsVectorParams) IsWrapType(_ llvm.Context, _ llvm.Type, typ llvm.Type, index int) bool { + return index > 0 && typ.TypeKind() == llvm.VectorTypeKind && p.td.TypeSizeInBits(typ) == 128 +} + +func (p *windowsVectorParams) GetTypeInfo(ctx llvm.Context, ft, typ llvm.Type, index int) *TypeInfo { + info := &TypeInfo{Type: typ, Type1: typ} + if p.IsWrapType(ctx, ft, typ, index) { + info.Kind = AttrPointer + info.Type1 = llvm.PointerType(typ, 0) + info.Size, info.Align, info.ByValAlign = 16, 16, 16 + } + return info +} diff --git a/internal/cabi/windows_vector_test.go b/internal/cabi/windows_vector_test.go new file mode 100644 index 0000000000..df16460535 --- /dev/null +++ b/internal/cabi/windows_vector_test.go @@ -0,0 +1,73 @@ +package cabi + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llgo/ssa" + "github.com/xgo-dev/llvm" +) + +func TestWindowsVectorParameterABI(t *testing.T) { + const ir = ` +declare <4 x float> @external(<4 x float>) +declare fastcc <4 x float> @fast(<4 x float>) +declare <4 x float> @llvm.sqrt.v4f32(<4 x float>) +define <4 x float> @forward(<4 x float> %x, ptr %indirect) { + %a = call <4 x float> @external(<4 x float> %x) + %b = call <4 x float> %indirect(<4 x float> %a) + %c = call fastcc <4 x float> @fast(<4 x float> %b) + %d = call <4 x float> @llvm.sqrt.v4f32(<4 x float> %c) + ret <4 x float> %d +} +` + llvm.InitializeAllTargets() + llvm.InitializeAllTargetMCs() + llvm.InitializeAllTargetInfos() + for _, goos := range []string{"windows", "linux"} { + t.Run(goos, func(t *testing.T) { + ctx := llvm.NewContext() + defer ctx.Dispose() + path := filepath.Join(t.TempDir(), "vector.ll") + if err := os.WriteFile(path, []byte(ir), 0600); err != nil { + t.Fatal(err) + } + buf, err := llvm.NewMemoryBufferFromFile(path) + if err != nil { + t.Fatal(err) + } + mod, err := ctx.ParseIR(buf) + if err != nil { + t.Fatal(err) + } + defer mod.Dispose() + prog := ssa.NewProgram(&ssa.Target{GOOS: goos, GOARCH: "amd64"}) + defer prog.Dispose() + tr := NewTransformer(prog, "", "", true) + tr.LowerWindowsVectorParams(mod) + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + body := mod.NamedFunction("forward").String() + if goos == "windows" { + if mod.NamedFunction("forward").Param(0).Type().TypeKind() != llvm.PointerTypeKind || + !strings.Contains(body, "load <4 x float>") || !strings.Contains(body, "call <4 x float> @external(ptr") || + !strings.Contains(body, "call <4 x float> %1(ptr") { + t.Fatalf("indirect vector ABI not exposed:\n%s", body) + } + } else if mod.NamedFunction("forward").Param(0).Type().TypeKind() != llvm.VectorTypeKind { + t.Fatalf("changed SysV vector ABI:\n%s", body) + } + for _, name := range []string{"fast", "llvm.sqrt.v4f32"} { + if mod.NamedFunction(name).Param(0).Type().TypeKind() != llvm.VectorTypeKind { + t.Fatalf("changed non-C ABI: %s", name) + } + } + if mod.NamedFunction("forward").GlobalValueType().ReturnType().TypeKind() != llvm.VectorTypeKind { + t.Fatal("changed vector return ABI") + } + }) + } +} diff --git a/internal/llvmfmv/comdat_test.go b/internal/llvmfmv/comdat_test.go new file mode 100644 index 0000000000..e7383d37ea --- /dev/null +++ b/internal/llvmfmv/comdat_test.go @@ -0,0 +1,50 @@ +package llvmfmv + +import ( + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestVariantCOMDAT(t *testing.T) { + for _, selection := range []string{"any", "samesize"} { + t.Run(selection, func(t *testing.T) { + mod := parseModule(t, ` +target triple = "x86_64-pc-windows-msvc" +$generic = comdat `+selection+` +declare i1 @avx2() "llgo.cpu.query"="x86.avx2" +declare float @external(float) "llgo.fmv.avx2-entry" +define linkonce float @generic(float %x) comdat { + %enabled = call i1 @avx2() + br i1 %enabled, label %fast, label %slow +fast: + %y = fadd float %x, %x + ret float %y +slow: + ret float %x +} +define float @plain(float %x) "llgo.fmv.avx2-entry" { + ret float %x +} +`) + original := mod.NamedFunction("generic").Comdat() + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + variant := mod.NamedFunction("generic" + variantSuffix) + cd := variant.Comdat() + if cd.C == nil || cd.C == original.C || cd.SelectionKind() != original.SelectionKind() || + cd.C != mod.Comdat(variant.Name()).C { + t.Fatalf("variant must have its own COMDAT with the original selection kind:\n%s", mod.String()) + } + for _, name := range []string{"generic" + baselineSuffix, "generic" + resolverSuffix, "plain" + variantSuffix, "external" + variantSuffix} { + if mod.NamedFunction(name).Comdat().C != nil { + t.Fatalf("%s gained an unnecessary COMDAT", name) + } + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + }) + } +} diff --git a/internal/llvmfmv/fmv.go b/internal/llvmfmv/fmv.go new file mode 100644 index 0000000000..3a9440023b --- /dev/null +++ b/internal/llvmfmv/fmv.go @@ -0,0 +1,359 @@ +// Package llvmfmv implements early, CPU-guarded LLVM function specialization +// directly through the LLVM Go API, independently of optimization and LTO. +package llvmfmv + +import ( + "fmt" + "strings" + + "github.com/xgo-dev/llvm" +) + +const ( + attributePrefix = "llgo.fmv." + symbolPrefix = ".__llgo_fmv_" + + // Shared with the frontend that emits the FMV attributes and CPU queries. + EntryAttribute = attributePrefix + "avx2-entry" + InlineEntryAttribute = attributePrefix + "inline-entry" + QueryAttribute = "llgo.cpu.query" + AVX2Query = "x86.avx2" + + doneAttribute = attributePrefix + "processed" + variantSuffix = symbolPrefix + "avx2" + baselineSuffix = symbolPrefix + "baseline" + resolverSuffix = symbolPrefix + "resolve" + slotSuffix = symbolPrefix + "slot" + featureSymbol = "github.com/xgo-dev/llgo/runtime/internal/runtime.CPUFeatures" + // Keep these bits in sync with runtime/_patch/internal/cpu/init_llgo.go. + featureInitialized = 1 + featureAVX2 = 2 +) + +func stringAttribute(fn llvm.Value, name string) string { + if attr := fn.GetStringAttributeAtIndex(-1, name); !attr.IsNil() { + return attr.GetStringValue() + } + return "" +} + +func hasAttribute(fn llvm.Value, name string) bool { + return !fn.GetStringAttributeAtIndex(-1, name).IsNil() +} + +func isQuery(fn llvm.Value) bool { + if fn.IsNil() || fn.IsAFunction().IsNil() || stringAttribute(fn, QueryAttribute) != AVX2Query { + return false + } + typ := fn.GlobalValueType() + ret := typ.ReturnType() + if ret.TypeKind() != llvm.IntegerTypeKind || ret.IntTypeWidth() != 1 || typ.IsFunctionVarArg() { + return false + } + // Before ABI lowering the Go method can still have its zero-sized receiver. + params := typ.ParamTypes() + return len(params) == 0 || len(params) == 1 && params[0].TypeKind() == llvm.StructTypeKind && params[0].StructElementTypesCount() == 0 +} + +// The first implementation preserves the existing SIMD128 ABI. Wider vectors +// and aggregates require their own target-independent boundary lowering. +func supportedType(typ llvm.Type) bool { + switch typ.TypeKind() { + case llvm.VoidTypeKind, llvm.IntegerTypeKind, llvm.PointerTypeKind, + llvm.FloatTypeKind, llvm.DoubleTypeKind, llvm.X86_FP80TypeKind, + llvm.FP128TypeKind, llvm.PPC_FP128TypeKind: + return true + case llvm.VectorTypeKind: + elem := typ.ElementType() + var width int + switch elem.TypeKind() { + case llvm.IntegerTypeKind: + width = elem.IntTypeWidth() + case llvm.FloatTypeKind: + width = 32 + case llvm.DoubleTypeKind: + width = 64 + default: + return false + } + return typ.VectorSize() <= 128/width + } + return false +} + +func supported(fn llvm.Value) bool { + typ := fn.GlobalValueType() + if typ.IsFunctionVarArg() || !supportedType(typ.ReturnType()) { + return false + } + for _, name := range []string{"naked", "returns_twice"} { + if !fn.GetEnumFunctionAttribute(llvm.AttributeKindID(name)).IsNil() { + return false + } + } + for _, typ := range typ.ParamTypes() { + if !supportedType(typ) { + return false + } + } + return true +} + +// Run preserves baseline entries and creates AVX2 implementations of supported +// SIMD128 functions. It runs after Win64 vector-parameter lowering but before +// aggregate ABI lowering and target optimization, +// including in O0 and ModeGen builds. baseline is the resolved compile-time +// GOAMD64 level, never a host environment read. Running it again is harmless. +func Run(mod llvm.Module, baseline string) error { + arch, _, _ := strings.Cut(mod.Target(), "-") + if arch != "x86_64" && arch != "amd64" && arch != "x86_64h" { + return nil + } + if baseline != "" && baseline != "v1" && baseline != "v2" && baseline != "v3" && baseline != "v4" { + return fmt.Errorf("SIMD FMV invalid GOAMD64 baseline: %s", baseline) + } + // Discover roots from direct uses of trusted queries. Ordinary packages + // without SIMD must not pay for a scan of every instruction in the module. + roots := make(map[llvm.Value]llvm.Value) + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if !isQuery(fn) { + continue + } + for use := fn.FirstUse(); !use.IsNil(); use = use.NextUse() { + call := use.User() + if !call.IsACallInst().IsNil() && call.CalledValue() == fn { + roots[call.InstructionParent().Parent()] = fn + } + } + } + if baseline == "v3" || baseline == "v4" { + // The official CPU policy does not allow GODEBUG to disable features + // required by GOAMD64. No version selection or extra helper entry is + // needed once AVX2 is already part of the compile-time contract. + for fn := range roots { + specialize(mod.Context(), fn, nil, true) + } + return nil + } + var originals []llvm.Value + for fn := mod.FirstFunction(); !fn.IsNil(); fn = llvm.NextFunction(fn) { + if hasAttribute(fn, doneAttribute) { + continue + } + if !hasAttribute(fn, EntryAttribute) && roots[fn].IsNil() { + continue + } + if !supported(fn) { + continue + } + suffixes := []string{variantSuffix} + if !roots[fn].IsNil() { + suffixes = append(suffixes, baselineSuffix, resolverSuffix, slotSuffix) + } + for _, suffix := range suffixes { + name := fn.Name() + suffix + // Check all names before modifying any function in the module. + if !mod.NamedValue(name).IsNil() { + return fmt.Errorf("SIMD FMV symbol collision: %s", name) + } + } + originals = append(originals, fn) + } + if len(originals) == 0 { + return nil + } + if mask := mod.NamedValue(featureSymbol); !mask.IsNil() && + (mask.IsAGlobalVariable().IsNil() || mask.GlobalValueType() != mod.Context().Int64Type()) { + return fmt.Errorf("SIMD FMV symbol collision: %s", featureSymbol) + } + + ctx := mod.Context() + source := newSourceInfo(mod) + variants := make(map[llvm.Value]llvm.Value, len(originals)) + baselines := make(map[llvm.Value]llvm.Value, len(roots)) + for _, fn := range originals { + variant := fn.CloneFunction() + variant.SetName(fn.Name() + variantSuffix) + if !fn.IsDeclaration() { + // CloneFunction preserves linkage but drops COMDAT. Give the + // variant its own group so generic instances merge independently + // of their original definitions when linking COFF objects. + if original := fn.Comdat(); original.C != nil { + group := mod.Comdat(variant.Name()) + group.SetSelectionKind(original.SelectionKind()) + variant.SetComdat(group) + } + source.clone(fn, variant) + } + features := stringAttribute(fn, "target-features") + if features != "" { + features += "," + } + variant.AddFunctionAttr(ctx.CreateStringAttribute("target-features", features+"+avx,+avx2")) + variant.RemoveStringAttributeAtIndex(-1, EntryAttribute) + variant.RemoveStringAttributeAtIndex(-1, InlineEntryAttribute) + variant.AddFunctionAttr(ctx.CreateStringAttribute(doneAttribute, "")) + fn.AddFunctionAttr(ctx.CreateStringAttribute(doneAttribute, "")) + variants[fn] = variant + if !roots[fn].IsNil() { + baseline := fn.CloneFunction() + baseline.SetName(fn.Name() + baselineSuffix) + baseline.SetLinkage(llvm.InternalLinkage) + baseline.RemoveStringAttributeAtIndex(-1, EntryAttribute) + baseline.RemoveStringAttributeAtIndex(-1, InlineEntryAttribute) + source.clone(fn, baseline) + baselines[fn] = baseline + } + } + for _, fn := range originals { + if fn.IsDeclaration() { + continue + } + variant := variants[fn] + specialize(ctx, variant, variants, true) + if baseline := baselines[fn]; !baseline.IsNil() { + specialize(ctx, baseline, baselines, false) + dispatch(mod, source, fn, baseline, variant) + } + } + return nil +} + +func specialize(ctx llvm.Context, fn llvm.Value, variants map[llvm.Value]llvm.Value, enabled bool) { + var queries []llvm.Value + for bb := fn.FirstBasicBlock(); !bb.IsNil(); bb = llvm.NextBasicBlock(bb) { + for inst := bb.FirstInstruction(); !inst.IsNil(); inst = llvm.NextInstruction(inst) { + if inst.IsACallInst().IsNil() && inst.IsAInvokeInst().IsNil() { + continue + } + callee := inst.CalledValue() + if isQuery(callee) { + if !inst.IsACallInst().IsNil() { + queries = append(queries, inst) + } + } else if variant := variants[callee]; !variant.IsNil() { + inst.SetOperand(inst.OperandsCount()-1, variant) + } + } + } + for _, query := range queries { + var bit uint64 + if enabled { + bit = 1 + } + query.ReplaceAllUsesWith(llvm.ConstInt(ctx.Int1Type(), bit, false)) + query.EraseFromParentAsInstruction() + } + // Mandatory legality cleanup, even at O0. These are local LLVM IR utilities, + // not a registered pass or an optional optimization pipeline. Iterate for + // conditions propagated through PHIs and short-circuit boolean expressions. + for { + changed := false + for bb := fn.FirstBasicBlock(); !bb.IsNil(); bb = llvm.NextBasicBlock(bb) { + if bb.SimplifyInstructionsInBlock() { + changed = true + } + if bb.ConstantFoldTerminator(true) { + changed = true + } + } + if fn.RemoveUnreachableBlocks() { + changed = true + } + if !changed { + break + } + } +} + +func dispatch(mod llvm.Module, source *sourceInfo, fn, baseline, variant llvm.Value) { + ctx := mod.Context() + resolver := fn.CloneFunction() + resolver.SetName(fn.Name() + resolverSuffix) + resolver.SetLinkage(llvm.InternalLinkage) + resolver.RemoveStringAttributeAtIndex(-1, EntryAttribute) + resolver.RemoveStringAttributeAtIndex(-1, InlineEntryAttribute) + resolver.RemoveEnumAttributeAtIndex(-1, llvm.AttributeKindID("alwaysinline")) + resolver.AddFunctionAttr(ctx.CreateEnumAttribute(llvm.AttributeKindID("noinline"), 0)) + source.cloneEntry(fn, resolver) + + mask := mod.NamedGlobal(featureSymbol) + if mask.IsNil() { + mask = llvm.AddGlobal(mod, ctx.Int64Type(), featureSymbol) + mask.SetAlignment(8) + } + slot := llvm.AddGlobal(mod, fn.Type(), fn.Name()+slotSuffix) + slot.SetLinkage(llvm.InternalLinkage) + slot.SetInitializer(resolver) + slot.SetAlignment(8) + + b := ctx.NewBuilder() + defer b.Dispose() + b.SetInsertPointAtEnd(ctx.InsertBasicBlock(fn.EntryBasicBlock(), "fmv.entry")) + // The ABI thunk is not another source execution frame. LLVM propagates the + // real callsite (including its outer inline chain) when inlining a nodebug + // callee. Keep source subprograms on the physical implementations and retain + // the public funcinfo identity for function values and sampled entry PCs. + // This gives the same synthetic-scope policy before and during LTO without + // requiring a compiler-specific late pass in the native linker. + fn.SetSubprogram(llvm.Metadata{}) + if hasAttribute(fn, InlineEntryAttribute) { + fn.RemoveEnumAttributeAtIndex(-1, llvm.AttributeKindID("noinline")) + fn.RemoveStringAttributeAtIndex(-1, InlineEntryAttribute) + } + target := b.CreateLoad(fn.Type(), slot, "target") + target.SetAlignment(8) + target.SetOrdering(llvm.AtomicOrderingMonotonic) + tailReturn(b, fn, target) + fn.RemoveUnreachableBlocks() + + old := resolver.EntryBasicBlock() + entry := ctx.InsertBasicBlock(old, "fmv.resolve") + uninitialized := ctx.InsertBasicBlock(old, "fmv.uninitialized") + selectImpl := ctx.InsertBasicBlock(old, "fmv.select") + b.SetInsertPointAtEnd(entry) + if sp := resolver.Subprogram(); !sp.IsNil() { + b.SetCurrentDebugLocation(0, 0, sp, llvm.Metadata{}) + } + // internal/cpu publishes the mask once before initializing its consumers. + // Pre-publication bootstrap calls are sequential and must not cache their + // fallback. After publication the mask is immutable; only the per-function + // slot can be updated concurrently and therefore needs atomic accesses. + features := b.CreateLoad(ctx.Int64Type(), mask, "features") + features.SetAlignment(8) + initialized := b.CreateAnd(features, llvm.ConstInt(ctx.Int64Type(), featureInitialized, false), "") + b.CreateCondBr(b.CreateICmp(llvm.IntNE, initialized, llvm.ConstInt(ctx.Int64Type(), 0, false), ""), selectImpl, uninitialized) + b.SetInsertPointAtEnd(uninitialized) + // Do not cache a pre-initialization fallback. Keep only this edge opaque: + // the resolver must remain tail-only, while ordinary callers may inline + // either implementation when their target features are compatible. + call := tailReturn(b, resolver, baseline) + call.AddCallSiteAttribute(-1, ctx.CreateEnumAttribute(llvm.AttributeKindID("noinline"), 0)) + b.SetInsertPointAtEnd(selectImpl) + enabled := b.CreateAnd(features, llvm.ConstInt(ctx.Int64Type(), featureAVX2, false), "") + check := b.CreateICmp(llvm.IntNE, enabled, llvm.ConstInt(ctx.Int64Type(), 0, false), "") + selected := b.CreateSelect(check, variant, baseline, "selected") + publish := b.CreateStore(selected, slot) + publish.SetAlignment(8) + publish.SetOrdering(llvm.AtomicOrderingMonotonic) + tailReturn(b, resolver, selected) + resolver.RemoveUnreachableBlocks() +} + +func tailReturn(b llvm.Builder, fn, target llvm.Value) llvm.Value { + call := b.CreateCall(fn.GlobalValueType(), target, fn.Params(), "") + call.SetInstructionCallConv(fn.FunctionCallConv()) + // Forward parameter/return ABI attributes, not function optimization policy. + for index := 0; index <= fn.ParamsCount(); index++ { + for _, attr := range fn.GetAttributesAtIndex(index) { + call.AddCallSiteAttribute(index, attr) + } + } + call.SetTailCallKind(llvm.TailCallKindMustTail) + if fn.GlobalValueType().ReturnType().TypeKind() == llvm.VoidTypeKind { + b.CreateRetVoid() + } else { + b.CreateRet(call) + } + return call +} diff --git a/internal/llvmfmv/fmv_test.go b/internal/llvmfmv/fmv_test.go new file mode 100644 index 0000000000..e71cf9df03 --- /dev/null +++ b/internal/llvmfmv/fmv_test.go @@ -0,0 +1,244 @@ +package llvmfmv + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func parseModule(t *testing.T, source string) llvm.Module { + t.Helper() + ctx := llvm.NewContext() + t.Cleanup(ctx.Dispose) + path := filepath.Join(t.TempDir(), "fmv.ll") + if err := os.WriteFile(path, []byte(source), 0600); err != nil { + t.Fatal(err) + } + buf, err := llvm.NewMemoryBufferFromFile(path) + if err != nil { + t.Fatal(err) + } + mod, err := ctx.ParseIR(buf) + if err != nil { + t.Fatal(err) + } + t.Cleanup(mod.Dispose) + return mod +} + +const fixture = ` +target triple = "x86_64-unknown-linux-gnu" +@address = global ptr @root +declare i1 @avx2() "llgo.cpu.query"="x86.avx2" +declare i1 @fma() "llgo.cpu.query"="x86.fma" +declare <4 x float> @external(<4 x float>) "llgo.fmv.avx2-entry" +declare <4 x float> @assembly(<4 x float>) +declare void @consume(ptr) +define void @addressOnly() { + call void @consume(ptr @avx2) + ret void +} +define <4 x float> @helper(<4 x float> %x) "llgo.fmv.avx2-entry" { + %y = call <4 x float> @external(<4 x float> %x) + ret <4 x float> %y +} +define <4 x float> @root(<4 x float> %x, ptr %indirect) "disable-tail-calls"="true" { + %ok = call i1 @avx2() + %no = xor i1 %ok, true + br i1 %no, label %fallback, label %fast +fast: + call void asm sideeffect "__llgo_pcsite_000000000000002a_$\7B:uid\7D:\0A.quad 0x000000000000002a", ""() + %y = call <4 x float> @helper(<4 x float> %x) + %z = call <4 x float> %indirect(<4 x float> %y) + %a = call <4 x float> @assembly(<4 x float> %z) + %f = call i1 @fma() + br i1 %f, label %hasfma, label %nofma +hasfma: + ret <4 x float> %a +nofma: + ret <4 x float> %z +fallback: + ret <4 x float> zeroinitializer +} +define <4 x float> @recursive(<4 x float> %x, i32 %n) "llgo.fmv.avx2-entry" { + %end = icmp eq i32 %n, 0 + br i1 %end, label %done, label %again +again: + %next = sub i32 %n, 1 + %r = call <4 x float> @recursive(<4 x float> %x, i32 %next) + ret <4 x float> %r +done: + ret <4 x float> %x +} +define <8 x float> @wide(<8 x float> %x) "llgo.fmv.avx2-entry" { + %ok = call i1 @avx2() + ret <8 x float> %x +} +!llgo.funcinfo = !{!0} +!llgo.pcline = !{!1} +!0 = !{i32 1, !"root", !"example.root", !"example.go", i32 10, i32 1} +!1 = !{i32 1, i64 42, !"root", !"example.go", i32 12, i32 1} +` + +func TestEarlySpecialization(t *testing.T) { + mod := parseModule(t, fixture) + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("%v\n%s", err, mod.String()) + } + root := mod.NamedFunction("root").String() + baseline := mod.NamedFunction("root.__llgo_fmv_baseline").String() + resolver := mod.NamedFunction("root.__llgo_fmv_resolve").String() + variant := mod.NamedFunction("root.__llgo_fmv_avx2").String() + if !strings.Contains(mod.NamedGlobal("address").String(), "ptr @root") || strings.Contains(mod.NamedGlobal("address").String(), "__llgo_fmv") { + t.Fatal("function address no longer names baseline entry") + } + for _, want := range []string{"load atomic ptr, ptr @root.__llgo_fmv_slot monotonic", "musttail call <4 x float> %target"} { + if !strings.Contains(root, want) { + t.Errorf("dispatcher missing %q:\n%s", want, root) + } + } + if strings.Contains(root, "@avx2(") || strings.Contains(baseline, "@avx2(") || !strings.Contains(baseline, "ret <4 x float> zeroinitializer") { + t.Fatalf("dispatcher or baseline retains an effective query:\n%s\n%s", root, baseline) + } + for _, want := range []string{"fmv.uninitialized:", "musttail call <4 x float> @root.__llgo_fmv_baseline", "store atomic ptr %selected, ptr @root.__llgo_fmv_slot monotonic"} { + if !strings.Contains(resolver, want) { + t.Errorf("resolver missing %q:\n%s", want, resolver) + } + } + for _, want := range []string{"@helper.__llgo_fmv_avx2(", "@fma()", "%indirect(", "@assembly("} { + if !strings.Contains(variant, want) { + t.Errorf("variant missing %q:\n%s", want, variant) + } + } + for _, unwanted := range []string{"@avx2()", "fallback:", "000000000000002a"} { + if strings.Contains(variant, unwanted) { + t.Errorf("variant retains %q:\n%s", unwanted, variant) + } + } + if !strings.Contains(mod.NamedFunction("helper.__llgo_fmv_avx2").String(), "@external.__llgo_fmv_avx2(") || + !strings.Contains(mod.NamedFunction("recursive.__llgo_fmv_avx2").String(), "@recursive.__llgo_fmv_avx2(") { + t.Fatal("direct call propagation missing") + } + if !mod.NamedFunction("wide.__llgo_fmv_avx2").IsNil() || !mod.NamedFunction("assembly.__llgo_fmv_avx2").IsNil() { + t.Fatal("unsupported ABI or unpromised external entry specialized") + } + if !mod.NamedFunction("addressOnly.__llgo_fmv_avx2").IsNil() { + t.Fatal("taking a query's address must not establish a CPU guard") + } + for _, want := range []string{`!"root.__llgo_fmv_avx2", !"example.root"`, `!"root.__llgo_fmv_avx2", !"example.go"`, `"target-features"="+avx,+avx2"`} { + if !strings.Contains(mod.String(), want) { + t.Errorf("missing %s", want) + } + } + before := mod.String() + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + if mod.String() != before { + t.Fatal("FMV is not idempotent") + } +} + +func TestOtherTargetUnchanged(t *testing.T) { + mod := parseModule(t, strings.Replace(fixture, "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", 1)) + before := mod.String() + if err := Run(mod, "v1"); err != nil || mod.String() != before { + t.Fatalf("non-amd64 module changed: %v", err) + } +} + +func TestSymbolCollision(t *testing.T) { + for _, symbol := range []string{ + "declare void @root.__llgo_fmv_avx2()", + "@root.__llgo_fmv_avx2 = global i8 0", + "@root.__llgo_fmv_avx2 = alias <4 x float> (<4 x float>, ptr), ptr @root", + "@root.__llgo_fmv_baseline = global i8 0", + "declare void @root.__llgo_fmv_resolve()", + "@root.__llgo_fmv_slot = global ptr null", + "@\"github.com/xgo-dev/llgo/runtime/internal/runtime.CPUFeatures\" = global i8 0", + } { + mod := parseModule(t, fixture+"\n"+symbol+"\n") + before := mod.String() + if err := Run(mod, "v1"); err == nil || !strings.Contains(err.Error(), "symbol collision") { + t.Fatalf("expected collision diagnostic for %s, got %v", symbol, err) + } + if mod.String() != before { + t.Fatal("collision partially mutated module") + } + } +} + +func TestUnsupportedSignaturesUnchanged(t *testing.T) { + for name, function := range map[string]string{ + "aggregate_parameter": `define i32 @unsupported({i32} %x) { + %q = call i1 @avx2() + ret i32 0 +}`, + "aggregate_result": `define {i32} @unsupported() { + %q = call i1 @avx2() + ret {i32} zeroinitializer +}`, + "wide_parameter": `define void @unsupported(<8 x float> %x) { + %q = call i1 @avx2() + ret void +}`, + "variadic": `define void @unsupported(i32 %n, ...) { + %q = call i1 @avx2() + ret void +}`, + "naked": `declare void @unsupported() naked "llgo.fmv.avx2-entry"`, + "returns_twice": `declare i32 @unsupported(ptr) returns_twice "llgo.fmv.avx2-entry"`, + } { + t.Run(name, func(t *testing.T) { + mod := parseModule(t, `target triple = "x86_64-unknown-linux-gnu" +declare i1 @avx2() "llgo.cpu.query"="x86.avx2" +`+function) + before := mod.String() + if err := Run(mod, "v1"); err != nil || mod.String() != before { + t.Fatalf("unsupported ABI was changed: %v\n%s", err, mod.String()) + } + }) + } +} + +func TestZeroSizedReceiverAndVoidDispatch(t *testing.T) { + mod := parseModule(t, ` +target triple = "x86_64-unknown-linux-gnu" +declare fastcc i1 @avx2({}) "llgo.cpu.query"="x86.avx2" +define fastcc void @root(ptr nonnull %p) { + %ok = call fastcc i1 @avx2({} zeroinitializer) + br i1 %ok, label %fast, label %fallback +fast: + store i32 1, ptr %p + ret void +fallback: + store i32 0, ptr %p + ret void +} +`) + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + root := mod.NamedFunction("root").String() + variant := mod.NamedFunction("root.__llgo_fmv_avx2").String() + for _, want := range []string{ + "load atomic ptr, ptr @root.__llgo_fmv_slot monotonic", + "musttail call fastcc void %target(ptr nonnull %p)", + } { + if !strings.Contains(root, want) { + t.Fatalf("dispatch lost receiver or call ABI %q:\n%s", want, root) + } + } + if strings.Contains(variant, "@avx2(") || strings.Contains(variant, "store i32 0") || !strings.Contains(variant, "store i32 1") { + t.Fatalf("zero-sized receiver query did not specialize:\n%s", variant) + } +} diff --git a/internal/llvmfmv/policy_test.go b/internal/llvmfmv/policy_test.go new file mode 100644 index 0000000000..4504c50a65 --- /dev/null +++ b/internal/llvmfmv/policy_test.go @@ -0,0 +1,275 @@ +package llvmfmv + +import ( + "strings" + "testing" + + "github.com/xgo-dev/llvm" +) + +func TestLazyDispatcherPolicy(t *testing.T) { + mod := parseModule(t, fixture) + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + if mod.NamedGlobal("root.__llgo_fmv_slot").IsNil() { + t.Fatal("dispatcher must load a cached implementation, initially the resolver") + } + if mod.NamedFunction("root.__llgo_fmv_baseline").IsNil() { + t.Fatal("baseline body must have its own implementation identity") + } + resolver := mod.NamedFunction("root.__llgo_fmv_resolve") + if resolver.IsNil() || resolver.GetEnumFunctionAttribute(llvm.AttributeKindID("noinline")).IsNil() { + t.Fatal("only the lazy resolver needs a synthetic noinline boundary") + } + if mod.NamedGlobal("root"+slotSuffix).Initializer() != resolver { + t.Fatal("first call does not enter the resolver") + } + for bb := resolver.FirstBasicBlock(); !bb.IsNil(); bb = llvm.NextBasicBlock(bb) { + if bb.AsValue().Name() != "fmv.uninitialized" { + continue + } + calls := 0 + for inst := bb.FirstInstruction(); !inst.IsNil(); inst = llvm.NextInstruction(inst) { + if !inst.IsAStoreInst().IsNil() { + t.Fatal("pre-initialization path cached a fallback") + } + if inst.IsACallInst().IsNil() { + continue + } + calls++ + if inst.CalledValue() != mod.NamedFunction("root"+baselineSuffix) || inst.TailCallKind() != llvm.TailCallKindMustTail || inst.GetCallSiteEnumAttribute(-1, llvm.AttributeKindID("noinline")).IsNil() { + t.Fatal("pre-initialization fallback must remain an opaque tail transfer") + } + } + if calls != 1 { + t.Fatalf("uninitialized resolver has %d calls, want one tail transfer", calls) + } + } + for _, name := range []string{"root", "root.__llgo_fmv_baseline", "root.__llgo_fmv_avx2"} { + if !mod.NamedFunction(name).GetEnumFunctionAttribute(llvm.AttributeKindID("noinline")).IsNil() { + t.Fatalf("%s must remain eligible for feature-compatible inlining", name) + } + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("%v\n%s", err, mod.String()) + } +} + +const inlineFixture = ` +target triple = "x86_64-unknown-linux-gnu" +declare i1 @avx2() "llgo.cpu.query"="x86.avx2" +define <4 x float> @helper(<4 x float> %x) "llgo.fmv.avx2-entry" "target-cpu"="x86-64" { + %y = fadd <4 x float> %x, %x + ret <4 x float> %y +} +define <4 x float> @root(<4 x float> %x) "target-cpu"="x86-64" { + %ok = call i1 @avx2() + br i1 %ok, label %fast, label %slow +fast: + %v = call <4 x float> @helper(<4 x float> %x) + ret <4 x float> %v +slow: + ret <4 x float> %x +} +define <4 x float> @caller(<4 x float> %x) noinline "target-cpu"="x86-64" { + %v = call <4 x float> @root(<4 x float> %x) + %r = fmul <4 x float> %v, %x + ret <4 x float> %r +} +` + +func TestCompileBaselineElidesVersions(t *testing.T) { + for _, baseline := range []string{"v3", "v4"} { + t.Run(baseline, func(t *testing.T) { + mod := parseModule(t, inlineFixture) + if err := Run(mod, baseline); err != nil { + t.Fatal(err) + } + if strings.Contains(mod.String(), "__llgo_fmv") || strings.Contains(mod.NamedFunction("root").String(), "@avx2(") || !mod.NamedGlobal(featureSymbol).IsNil() { + t.Fatalf("compile baseline retained redundant versioning:\n%s", mod.String()) + } + root := mod.NamedFunction("root").String() + if strings.Contains(root, "br i1") || strings.Contains(root, "slow:") || !strings.Contains(root, "call <4 x float> @helper(") { + t.Fatalf("compile baseline did not select the enabled branch:\n%s", root) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + }) + } +} + +func optimizePolicyModule(t *testing.T, mod llvm.Module, pipeline string) { + t.Helper() + llvm.InitializeAllTargetInfos() + llvm.InitializeAllTargets() + llvm.InitializeAllTargetMCs() + target, err := llvm.GetTargetFromTriple(mod.Target()) + if err != nil { + t.Fatal(err) + } + tm := target.CreateTargetMachine(mod.Target(), "x86-64", "", llvm.CodeGenLevelDefault, llvm.RelocDefault, llvm.CodeModelDefault) + defer tm.Dispose() + options := llvm.NewPassBuilderOptions() + defer options.Dispose() + options.SetVerifyEach(true) + if err := mod.RunPasses(pipeline, tm, options); err != nil { + t.Fatal(err) + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatalf("%v\n%s", err, mod.String()) + } +} + +func TestInliningPolicy(t *testing.T) { + for _, pipeline := range []string{"default", "thinlto-pre-link,thinlto", "lto-pre-link,lto"} { + t.Run(pipeline, func(t *testing.T) { + mod := parseModule(t, inlineFixture) + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + ctx := mod.Context() + b := ctx.NewBuilder() + defer b.Dispose() + helper := mod.NamedFunction("helper.__llgo_fmv_avx2") + for _, name := range []string{"compatible", "incompatible"} { + fn := llvm.AddFunction(mod, name, helper.GlobalValueType()) + fn.AddFunctionAttr(ctx.CreateStringAttribute("target-cpu", "x86-64")) + if name == "compatible" { + fn.AddFunctionAttr(ctx.CreateStringAttribute("target-features", "+avx,+avx2")) + } + b.SetInsertPointAtEnd(ctx.AddBasicBlock(fn, "entry")) + call := b.CreateCall(helper.GlobalValueType(), helper, fn.Params(), "") + b.CreateRet(b.CreateFMul(call, fn.Param(0), "")) + } + optimizePolicyModule(t, mod, pipeline) + caller := mod.NamedFunction("caller").String() + if strings.Contains(caller, "@root(") || strings.Contains(caller, "musttail") || !strings.Contains(caller, "fmul") { + t.Fatalf("dispatcher did not inline with its caller continuation preserved:\n%s", caller) + } + if strings.Contains(mod.NamedFunction("compatible").String(), "@helper.__llgo_fmv_avx2(") { + t.Fatal("feature-compatible caller failed to inline the implementation") + } + if !strings.Contains(mod.NamedFunction("incompatible").String(), "@helper.__llgo_fmv_avx2(") { + t.Fatal("AVX2 implementation was inlined into a baseline caller") + } + }) + } +} + +func TestSourceInliningAttributes(t *testing.T) { + for _, attr := range []string{"noinline", "alwaysinline"} { + t.Run(attr, func(t *testing.T) { + mod := parseModule(t, strings.Replace(inlineFixture, `@root(<4 x float> %x) "target-cpu"`, `@root(<4 x float> %x) `+attr+` "target-cpu"`, 1)) + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + for _, suffix := range []string{"", baselineSuffix, variantSuffix} { + if mod.NamedFunction("root" + suffix).GetEnumFunctionAttribute(llvm.AttributeKindID(attr)).IsNil() { + t.Fatalf("%s lost source %s policy", "root"+suffix, attr) + } + } + resolver := mod.NamedFunction("root" + resolverSuffix) + if !resolver.GetEnumFunctionAttribute(llvm.AttributeKindID("alwaysinline")).IsNil() || resolver.GetEnumFunctionAttribute(llvm.AttributeKindID("noinline")).IsNil() { + t.Fatal("resolver did not establish its own synthetic boundary") + } + if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil { + t.Fatal(err) + } + }) + } +} + +func TestPhysicalFrameDoesNotPinDispatcher(t *testing.T) { + for _, pipeline := range []string{"default", "thinlto-pre-link,thinlto", "lto-pre-link,lto"} { + t.Run(pipeline, func(t *testing.T) { + mod := parseModule(t, strings.Replace(inlineFixture, `@root(<4 x float> %x) "target-cpu"`, `@root(<4 x float> %x) noinline "disable-tail-calls"="true" "llgo.fmv.inline-entry" "target-cpu"`, 1)) + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + if !mod.NamedFunction("root").GetEnumFunctionAttribute(llvm.AttributeKindID("noinline")).IsNil() { + t.Fatal("physical-frame policy leaked onto the synthetic dispatcher") + } + for _, suffix := range []string{baselineSuffix, variantSuffix} { + impl := mod.NamedFunction("root" + suffix) + if impl.GetEnumFunctionAttribute(llvm.AttributeKindID("noinline")).IsNil() || stringAttribute(impl, "disable-tail-calls") != "true" { + t.Fatal("dispatcher inlining removed the required physical source frame") + } + } + optimizePolicyModule(t, mod, pipeline) + caller := mod.NamedFunction("caller").String() + if strings.Contains(caller, "@root(") || strings.Contains(caller, "musttail") || !strings.Contains(caller, "fmul") { + t.Fatalf("dispatcher did not inline while preserving its caller continuation:\n%s", caller) + } + }) + } +} + +func TestDispatcherInlineDebugIdentity(t *testing.T) { + mod := parseModule(t, ` +target triple = "x86_64-unknown-linux-gnu" +declare i1 @avx2() "llgo.cpu.query"="x86.avx2" +define float @root(float %x) "target-cpu"="x86-64" !dbg !4 { + %ok = call i1 @avx2(), !dbg !5 + br i1 %ok, label %fast, label %slow, !dbg !5 +fast: + %y = fadd float %x, %x, !dbg !5 + ret float %y, !dbg !5 +slow: + ret float %x, !dbg !5 +} +define float @middle(float %x) alwaysinline "target-cpu"="x86-64" !dbg !6 { + %v = call float @root(float %x), !dbg !7 + %r = fmul float %v, %x, !dbg !7 + ret float %r, !dbg !7 +} +define float @outer(float %x) noinline "target-cpu"="x86-64" !dbg !8 { + %v = call float @middle(float %x), !dbg !9 + %r = fadd float %v, %x, !dbg !9 + ret float %r, !dbg !9 +} +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!10, !11} +!llgo.funcinfo = !{!12} +!0 = distinct !DICompileUnit(language: DW_LANG_Go, file: !1, producer: "llgo", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug) +!1 = !DIFile(filename: "inline.go", directory: "/src") +!2 = !DISubroutineType(types: !3) +!3 = !{} +!4 = distinct !DISubprogram(name: "root", linkageName: "root", scope: !1, file: !1, line: 10, type: !2, scopeLine: 10, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0) +!5 = !DILocation(line: 11, column: 1, scope: !4) +!6 = distinct !DISubprogram(name: "middle", linkageName: "middle", scope: !1, file: !1, line: 20, type: !2, scopeLine: 20, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0) +!7 = !DILocation(line: 21, column: 1, scope: !6) +!8 = distinct !DISubprogram(name: "outer", linkageName: "outer", scope: !1, file: !1, line: 30, type: !2, scopeLine: 30, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0) +!9 = !DILocation(line: 31, column: 1, scope: !8) +!10 = !{i32 2, !"Dwarf Version", i32 4} +!11 = !{i32 2, !"Debug Info Version", i32 3} +!12 = !{i32 1, !"root", !"root", !"inline.go", i32 10, i32 1} +`) + if err := Run(mod, "v1"); err != nil { + t.Fatal(err) + } + optimizePolicyModule(t, mod, "default") + outer := mod.NamedFunction("outer") + middle := mod.NamedFunction("middle") + seen := false + for bb := outer.FirstBasicBlock(); !bb.IsNil(); bb = llvm.NextBasicBlock(bb) { + for inst := bb.FirstInstruction(); !inst.IsNil(); inst = llvm.NextInstruction(inst) { + if inst.IsALoadInst().IsNil() || inst.Operand(0) != mod.NamedGlobal("root"+slotSuffix) { + continue + } + seen = true + loc := inst.InstructionDebugLoc() + if loc.IsNil() || loc.LocationScope() != middle.Subprogram() || loc.LocationLine() != 21 { + t.Fatalf("synthetic dispatcher did not inherit its real caller location:\n%s", mod.String()) + } + parent := loc.LocationInlinedAt() + if parent.IsNil() || parent.LocationScope() != outer.Subprogram() || parent.LocationLine() != 31 || !parent.LocationInlinedAt().IsNil() { + t.Fatalf("outer inline chain was lost or gained a synthetic frame:\n%s", mod.String()) + } + } + } + if !seen { + t.Fatalf("dispatcher was not inlined into outer:\n%s", mod.String()) + } +} diff --git a/internal/llvmfmv/source.go b/internal/llvmfmv/source.go new file mode 100644 index 0000000000..ab420cd730 --- /dev/null +++ b/internal/llvmfmv/source.go @@ -0,0 +1,114 @@ +package llvmfmv + +import ( + "fmt" + "hash/fnv" + "strings" + + "github.com/xgo-dev/llvm" +) + +type sourceInfo struct { + mod llvm.Module + functions map[string][][]llvm.Value + lines map[string][][]llvm.Value + usedIDs map[uint64]bool +} + +// Runtime tables intentionally use strings, so LLVM's generic function clone +// cannot update them. Keep these layouts in sync with ssa/funcinfo.go: +// funcinfo = {version, symbol, Go name, file, line, column[, flags]} +// pcline = {version, id, symbol, file, line, column} +func newSourceInfo(mod llvm.Module) *sourceInfo { + s := &sourceInfo{mod: mod, functions: make(map[string][][]llvm.Value), + lines: make(map[string][][]llvm.Value), usedIDs: make(map[uint64]bool)} + for _, row := range mod.NamedMetadataOperands("llgo.funcinfo") { + fields := row.MDNodeOperands() + if len(fields) >= 6 && !fields[1].IsNil() && fields[1].IsAMDString() { + name := fields[1].MDString() + s.functions[name] = append(s.functions[name], fields) + } + } + for _, row := range mod.NamedMetadataOperands("llgo.pcline") { + fields := row.MDNodeOperands() + if len(fields) != 6 || fields[1].IsNil() || fields[1].IsAConstantInt().IsNil() { + continue + } + s.usedIDs[fields[1].ZExtValue()] = true + if !fields[2].IsNil() && fields[2].IsAMDString() { + name := fields[2].MDString() + s.lines[name] = append(s.lines[name], fields) + } + } + return s +} + +func metadataFields(fields []llvm.Value) []llvm.Metadata { + mds := make([]llvm.Metadata, len(fields)) + for i, value := range fields { + if !value.IsNil() { + mds[i] = value.AsMetadata() + } + } + return mds +} + +func (s *sourceInfo) cloneEntry(fn, variant llvm.Value) { + ctx := s.mod.Context() + name := variant.Name() + // LLVM's CloneFunction gives the implementation its own subprogram and + // remaps local debug scopes. Preserve the Go display name and source paths. + if sp := variant.Subprogram(); !sp.IsNil() { + sp.SetSubprogramLinkageName(name) + } + for _, fields := range s.functions[fn.Name()] { + mds := metadataFields(fields) + mds[1] = ctx.MDString(name) + s.mod.AddNamedMetadataOperand("llgo.funcinfo", ctx.MDNode(mds)) + } +} + +func (s *sourceInfo) clone(fn, variant llvm.Value) { + s.cloneEntry(fn, variant) + ctx := s.mod.Context() + name := variant.Name() + var replacements []string + for _, fields := range s.lines[fn.Name()] { + old := fields[1].ZExtValue() + oldHex := fmt.Sprintf("%016x", old) + hash := fnv.New64a() + hash.Write([]byte(name + ":" + oldHex)) + id := hash.Sum64() + for id == 0 || s.usedIDs[id] { + id++ + } + s.usedIDs[id] = true + mds := metadataFields(fields) + mds[1] = llvm.ConstInt(fields[1].Type(), id, false).AsMetadata() + mds[2] = ctx.MDString(name) + s.mod.AddNamedMetadataOperand("llgo.pcline", ctx.MDNode(mds)) + replacements = append(replacements, oldHex, fmt.Sprintf("%016x", id)) + } + if len(replacements) == 0 { + return + } + rewrite := strings.NewReplacer(replacements...) + for bb := variant.FirstBasicBlock(); !bb.IsNil(); bb = llvm.NextBasicBlock(bb) { + for inst := bb.FirstInstruction(); !inst.IsNil(); inst = llvm.NextInstruction(inst) { + if inst.IsACallInst().IsNil() && inst.IsAInvokeInst().IsNil() { + continue + } + callee := inst.CalledValue() + if callee.IsAInlineAsm().IsNil() { + continue + } + asm := callee.InlineAsmInfo() + if !strings.Contains(asm.Assembly, "__llgo_pcsite_") { + continue + } + cloned := llvm.InlineAsm(asm.Type, rewrite.Replace(asm.Assembly), asm.Constraints, + asm.HasSideEffects, asm.IsAlignStack, asm.Dialect, asm.CanThrow) + inst.SetOperand(inst.OperandsCount()-1, cloned) + } + } +} diff --git a/runtime/_patch/internal/cpu/init_llgo.go b/runtime/_patch/internal/cpu/init_llgo.go index a2980df5bd..f150b9ff15 100644 --- a/runtime/_patch/internal/cpu/init_llgo.go +++ b/runtime/_patch/internal/cpu/init_llgo.go @@ -10,7 +10,18 @@ import _ "unsafe" func init() { llgoPrepareCPU() Initialize(llgoCPUEnvironment()) + // Publish once, after the official detector has applied GODEBUG. Bit zero + // distinguishes initialization from an initialized CPU with AVX2 disabled. + // Keep this layout in sync with internal/llvmfmv's feature bits. + features := uint64(1) + if X86.HasAVX2 { + features |= 2 + } + llgoCPUFeatures = features } +//go:linkname llgoCPUFeatures github.com/xgo-dev/llgo/runtime/internal/runtime.CPUFeatures +var llgoCPUFeatures uint64 + //go:linkname llgoCPUEnvironment github.com/xgo-dev/llgo/runtime/internal/runtime.CPUEnvironment func llgoCPUEnvironment() string diff --git a/runtime/internal/runtime/cpu_features.go b/runtime/internal/runtime/cpu_features.go new file mode 100644 index 0000000000..2628339e6e --- /dev/null +++ b/runtime/internal/runtime/cpu_features.go @@ -0,0 +1,6 @@ +package runtime + +// CPUFeatures is published by internal/cpu after applying GODEBUG overrides. +// It is immutable after package initialization. Generated FMV resolvers use a +// baseline without caching while the initialized bit is still clear. +var CPUFeatures uint64 diff --git a/ssa/funcinfo.go b/ssa/funcinfo.go index fc1394ee85..0df8ae3b66 100644 --- a/ssa/funcinfo.go +++ b/ssa/funcinfo.go @@ -30,6 +30,9 @@ const ( FuncInfoFlagWrapper uint32 = 1 ) +// internal/llvmfmv/source.go also reads these versioned row layouts when cloning +// functions. Update its symbol and PC-site remapping when changing the schema. + // EnableFuncInfoMetadata controls emission of DCE-safe function source // metadata. The metadata intentionally stores symbol names as strings instead // of function pointer operands, so it can be consumed before materializing a diff --git a/ssa/simd_fmv.go b/ssa/simd_fmv.go new file mode 100644 index 0000000000..bd45b9cf4c --- /dev/null +++ b/ssa/simd_fmv.go @@ -0,0 +1,28 @@ +package ssa + +import "github.com/xgo-dev/llgo/internal/llvmfmv" + +// MarkSIMDFMV records compiler-checked source identities for the early LLVM +// transform. An available variant is a cross-package ABI promise: the defining +// package emits it, and guarded callers may refer to it without LTO. +func (f Function) MarkSIMDFMV(available bool, query string) { + if f.Prog.Target().GOARCH != "amd64" { + return + } + if available { + f.impl.AddFunctionAttr(f.Prog.ctx.CreateStringAttribute(llvmfmv.EntryAttribute, "")) + } + if query != "" { + f.impl.AddFunctionAttr(f.Prog.ctx.CreateStringAttribute(llvmfmv.QueryAttribute, query)) + } +} + +// AllowSIMDDispatcherInlining separates a compiler-required physical source +// frame from the synthetic FMV entry. The implementation retains its frame; +// the tail-only dispatcher may inline without changing the source call stack. +// Callers must not set this for an explicit noinline directive or -l. +func (f Function) AllowSIMDDispatcherInlining() { + if f.Prog.Target().GOARCH == "amd64" { + f.impl.AddFunctionAttr(f.Prog.ctx.CreateStringAttribute(llvmfmv.InlineEntryAttribute, "")) + } +} diff --git a/test/simd/README.md b/test/simd/README.md index f11add414a..7249cecb88 100644 --- a/test/simd/README.md +++ b/test/simd/README.md @@ -39,6 +39,75 @@ checks direct, indirect, deferred, and linkname calls. SIMD reflection, portable `simd` specialization, general FMV, and 256/512-bit vectors remain outside this implemented stage. +## CPU-guarded SIMD128 specialization + +On amd64, functions calling the official `archsimd.X86.AVX2` query get an +AVX2 version when their LLVM signature contains only scalars, pointers, and +vectors up to 128 bits. The original entry loads a cached implementation and +tail-forwards to it. A separate resolver selects the baseline or AVX2 body from +the immutable, post-`GODEBUG` CPU snapshot on first use. Before CPU initialization +it tail-forwards to the baseline without caching. Function addresses retain the +original entry. Both bodies fold the selected AVX2 query and remove dead branches +even at O0, before aggregate ABI lowering and target optimization. GOAMD64=v3/v4 +already require AVX2, so they fold the query without generating extra versions, +a resolver, or a cache slot. + +Direct calls from specialized code to eligible SIMD128 functions use matching +AVX2 versions, including across packages without LTO. Only functions with Go +bodies or compiler-generated SIMD intrinsic bodies promise those entries; +bodyless assembly declarations retain their original calls. Ordinary calls +and indirect calls retain the baseline entry. Aggregate signatures and +arbitrary feature combinations are not yet specialized. + +The transform is implemented in Go using the LLVM Go API. It discovers CPU +queries, clones eligible functions, rewrites direct calls and creates dispatch +blocks without registering a custom LLVM pass. Generic LLVM cloning and local +CFG utilities preserve instruction and debug metadata and remove dead branches +even at O0. The transform runs independently of the optimization level and LTO +plugin. Specialized functions retain their Go source identity and get distinct +runtime PC-line records. + +Inlining follows the same FMV policy as GoALLC: the public dispatcher and the +physical implementations remain eligible for normal, feature-compatible LLVM +inlining. The lazy resolver is noinline, and its pre-initialization baseline +edge has a call-site noinline attribute so it stays tail-only. Explicit source +noinline directives and disabled inlining remain honored. LLGo's existing +runtime-frame requirements remain on the physical body, not the synthetic +dispatcher. LLVM lowers a cloned musttail call to an ordinary call when the +dispatcher is inlined into a caller with a continuation. + +The dispatcher retains its runtime function identity but has no synthetic debug +subprogram. LLVM therefore attributes an inlined dispatcher to its real call +site, preserving outer inline chains without adding a second source execution +frame. The baseline/AVX2 implementations keep their own debug subprograms and +the original Go display name. These are FMV-specific identity rules; LLGo's +general runtime inline-frame representation remains separate from GoALLC's +GoObj inline-tree format. + +On Windows/amd64, the compiler exposes the native indirect SIMD128 parameter +ABI before creating dispatchers. Tail transfers forward the caller's parameter +storage, avoiding vector temporaries in a resolver frame that the jump releases. +Vector results keep their native register ABI. + +`GOAMD64` still controls the compilation baseline. The dispatcher observes +the post-`GODEBUG=cpu.*` AVX2 query; instruction capability does not imply +other observable query results. In particular, the AVX and FMA queries remain +dynamic when AVX2 is enabled. This does not implement portable `simd` width +selection or 256/512-bit vector calling conventions. + +Run the native matrix with: + +```sh +LLGO=/path/to/llgo bash dev/test_native_simd.sh +``` + +On amd64 it runs O0 without LTO and O2 without LTO, with ThinLTO, and with +Full LTO. Each binary runs the complete SIMD suite, then the FMV tests with +AVX2, AVX, FMA, and all optional CPU features disabled in separate processes. +The FMV tests cover initialization, direct and indirect entry calls, +cross-package calls, independent CPU queries, and source-level stack traces. +On other native SIMD targets the script retains the O0/O2 suite. + ## Running From the repository root, using the built LLGo binary on `PATH`: diff --git a/test/simd/fmv_amd64_test.go b/test/simd/fmv_amd64_test.go new file mode 100644 index 0000000000..d58e762843 --- /dev/null +++ b/test/simd/fmv_amd64_test.go @@ -0,0 +1,142 @@ +//go:build goexperiment.simd && amd64 + +package simd_test + +import ( + "runtime" + "simd/archsimd" + "strings" + "testing" + + "github.com/xgo-dev/llgo/test/simd/internal/vectorcall" +) + +//go:noinline +func fmvGuarded(x archsimd.Float32x4) archsimd.Float32x4 { + if archsimd.X86.AVX2() { + x = vectorcall.Echo(x, false) + // GODEBUG may disable FMA independently of AVX2. Instruction support + // must never replace the observable result of this separate query. + if archsimd.X86.FMA() { + x = x.Add(archsimd.BroadcastFloat32x4(3)) + } else { + x = x.Sub(archsimd.BroadcastFloat32x4(3)) + } + if !archsimd.X86.AVX() { + x = x.Add(archsimd.BroadcastFloat32x4(11)) + } + return x + } + return x.Sub(archsimd.BroadcastFloat32x4(7)) +} + +var fmvInitial = fmvGuarded(archsimd.BroadcastFloat32x4(10)) + +func TestSIMDFMVDispatch(t *testing.T) { + want := float32(3) + if archsimd.X86.AVX2() { + want = 9 + if archsimd.X86.FMA() { + want = 15 + } + if !archsimd.X86.AVX() { + want += 11 + } + } + t.Logf("AVX=%v AVX2=%v FMA=%v", archsimd.X86.AVX(), archsimd.X86.AVX2(), archsimd.X86.FMA()) + fn := fmvGuarded + for _, value := range []archsimd.Float32x4{fmvInitial, fmvGuarded(archsimd.BroadcastFloat32x4(10)), fn(archsimd.BroadcastFloat32x4(10))} { + var lanes [4]float32 + value.StoreArray(&lanes) + for _, got := range lanes { + if got != want { + t.Fatalf("guarded result %v, want %g", lanes, want) + } + } + } +} + +//go:noinline +func fmvTrace(x archsimd.Float32x4, pcs *[20]uintptr, n *int) archsimd.Float32x4 { + if archsimd.X86.AVX2() { + *n = runtime.Callers(0, pcs[:]) + return vectorcall.Echo(x, false) + } + *n = runtime.Callers(0, pcs[:]) + return x +} + +func TestSIMDFMVTraceback(t *testing.T) { + var pcs [20]uintptr + var n int + fmvTrace(archsimd.BroadcastFloat32x4(1), &pcs, &n) + checkFMVTrace(t, pcs[:n], "fmvTrace") +} + +// This entry must remain eligible for inlining. Its synthetic dispatcher can +// disappear into fmvInlineTraceCaller while its selected implementation still +// contributes exactly one source execution frame. +func fmvInlineTrace(x archsimd.Float32x4, pcs *[20]uintptr, n *int) archsimd.Float32x4 { + if archsimd.X86.AVX2() { + x = x.Add(archsimd.BroadcastFloat32x4(2)) + } else { + x = x.Sub(archsimd.BroadcastFloat32x4(2)) + } + *n = runtime.Callers(0, pcs[:]) + return x +} + +//go:noinline +func fmvInlineTraceCaller(x archsimd.Float32x4, pcs *[20]uintptr, n *int) archsimd.Float32x4 { + // The continuation must execute after the inlined dispatch. Keeping its + // result observable also prevents a tail transfer from removing this frame. + return fmvInlineTrace(x, pcs, n).Mul(archsimd.BroadcastFloat32x4(3)) +} + +func TestSIMDFMVInlineTraceback(t *testing.T) { + var pcs [20]uintptr + var n int + x := fmvInlineTraceCaller(archsimd.BroadcastFloat32x4(5), &pcs, &n) + var lanes [4]float32 + x.StoreArray(&lanes) + want := float32(9) + if archsimd.X86.AVX2() { + want = 21 + } + for _, got := range lanes { + if got != want { + t.Fatalf("inlined dispatcher continuation returned %g, want %g", got, want) + } + } + checkFMVTrace(t, pcs[:n], "fmvInlineTrace", "fmvInlineTraceCaller") +} + +func checkFMVTrace(t *testing.T, pcs []uintptr, names ...string) { + t.Helper() + frames := runtime.CallersFrames(pcs) + counts := make([]int, len(names)) + var trace []runtime.Frame + for { + frame, more := frames.Next() + trace = append(trace, frame) + if strings.Contains(frame.Function, "__llgo_fmv") { + t.Fatalf("compiler variant leaked into traceback: %+v", frame) + } + for i, name := range names { + if strings.HasSuffix(frame.Function, "."+name) { + counts[i]++ + if !strings.HasSuffix(frame.File, "fmv_amd64_test.go") || frame.Line == 0 { + t.Fatalf("missing source location: %+v", frame) + } + } + } + if !more { + break + } + } + for i, name := range names { + if counts[i] != 1 { + t.Fatalf("expected one %s frame, got %d; frames: %+v", name, counts[i], trace) + } + } +}