compiler: add early AVX2 SIMD128 multiversioning through Go APIs

.github/workflows/llgo.yml

@@ -315,14 +315,23 @@ jobs:
           (cd ./_demo/c/cargs && llgo run .)
 
       - name: Test SIMD behavior with Go and LLGo
+        id: native_simd
         if: matrix.windows_arch != '386'
         shell: bash
         env:
           GOEXPERIMENT: simd
+          LLGO_SIMD_ARTIFACT_DIR: ${{ runner.temp }}/llgo-simd-matrix
         run: |
           go test -count=1 -timeout=2m ./test/simd/...
-          llgo test -O0 -count=1 -timeout=2m ./test/simd/...
-          llgo test -O2 -count=1 -timeout=2m ./test/simd/...
+          bash dev/test_native_simd.sh
+
+      - name: Upload failed native SIMD executables
+        if: failure() && steps.native_simd.outcome == 'failure'
+        uses: actions/upload-artifact@v7
+        with:
+          name: native-simd-${{ matrix.os }}-${{ matrix.windows_abi || 'native' }}-${{ matrix.windows_arch || 'host' }}
+          path: ${{ runner.temp }}/llgo-simd-matrix
+          retention-days: 3
 
       - name: Test demos
         run: bash .github/workflows/test_demo.sh

cl/compile.go

@@ -674,6 +674,7 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun
 			fn = pkg.NewFuncEx(name, sig, background, false, p.needsLinkOnce(f))
 		}
 	}
+	p.markSIMDFMV(fn, f)
 	if p.prog.Target().GOARCH == "wasm" {
 		if decl, ok := f.Syntax().(*ast.FuncDecl); ok {
 			fullName := declarationFuncName(llssa.PathOf(pkgTypes), decl, f.Object(), f.Signature.Recv())
@@ -692,6 +693,9 @@ func (p *context) compileFuncDecl(pkg llssa.Package, f *ssa.Function) (llssa.Fun
 	if noInlineDirective || runtimeStackNoInline || pcLineNoInline || usesRecover {
 		fn.DisableTailCalls()
 	}
+	if !disableInline && !noInlineDirective && (runtimeStackNoInline || pcLineNoInline || usesRecover) {
+		fn.AllowSIMDDispatcherInlining()
+	}
 	p.funcs[f] = fn
 	isCgo := isCgoExternSymbol(f)
 	if (len(f.Blocks) != 0 || simdDecl) && p.prog.FuncInfoMetadataEnabled() {

cl/instr.go

@@ -735,6 +735,9 @@ func (p *context) funcOf(fn *ssa.Function) (aFn llssa.Function, pyFn llssa.PyObj
 				aFn.Inline(llssa.NoInline)
 			}
 		}
+		if ftype == goFunc {
+			p.markSIMDFMV(aFn, fn)
+		}
 	}
 	return
 }

cl/simd_fmv.go

@@ -0,0 +1,54 @@
+package cl
+
+import (
+	"go/types"
+
+	"github.com/xgo-dev/llgo/internal/llvmfmv"
+	llssa "github.com/xgo-dev/llgo/ssa"
+	"golang.org/x/tools/go/ssa"
+)
+
+func simdCPUQuery(fn *ssa.Function) string {
+	obj, ok := fn.Object().(*types.Func)
+	if !ok || obj.Pkg() == nil || obj.Pkg().Path() != "simd/archsimd" || obj.Name() != "AVX2" ||
+		!types.Identical(obj.Type(), fn.Signature) {
+		return ""
+	}
+	sig := fn.Signature
+	if sig.Recv() == nil || sig.Params().Len() != 0 || sig.Results().Len() != 1 ||
+		!types.Identical(sig.Results().At(0).Type(), types.Typ[types.Bool]) {
+		return ""
+	}
+	recv, ok := sig.Recv().Type().(*types.Named)
+	if !ok || recv.Obj().Pkg() != obj.Pkg() || recv.Obj().Name() != "X86Features" {
+		return ""
+	}
+	return llvmfmv.AVX2Query
+}
+
+func (p *context) markSIMDFMV(fn llssa.Function, source *ssa.Function) {
+	if fn == nil || p.prog.Target().GOARCH != "amd64" {
+		return
+	}
+	// Bodyless assembly/linkname declarations do not promise a compiler-emitted
+	// entry. Official SIMD intrinsic declarations have LLGo-generated bodies.
+	_, intrinsic := lookupSIMD(source, "amd64")
+	available := len(source.Blocks) != 0 || intrinsic
+	hasVector := false
+	sig := source.Signature
+	check := func(t types.Type) {
+		if _, ok := llssa.SIMDVectorShape(t); ok {
+			hasVector = true
+		}
+	}
+	if sig.Recv() != nil {
+		check(sig.Recv().Type())
+	}
+	for i := 0; i < sig.Params().Len(); i++ {
+		check(sig.Params().At(i).Type())
+	}
+	for i := 0; i < sig.Results().Len(); i++ {
+		check(sig.Results().At(i).Type())
+	}
+	fn.MarkSIMDFMV(available && hasVector, simdCPUQuery(source))
+}

cl/simd_test.go

@@ -13,6 +13,39 @@ import (
 	"golang.org/x/tools/go/ssa/ssautil"
 )
 
+func TestSIMDFMVQueryIdentity(t *testing.T) {
+	for _, tc := range []struct {
+		name, path, recv, signature string
+		want                        string
+	}{
+		{"official", "simd/archsimd", "X86Features", "AVX2() bool", "x86.avx2"},
+		{"other package", "example/archsimd", "X86Features", "AVX2() bool", ""},
+		{"other receiver", "simd/archsimd", "OtherFeatures", "AVX2() bool", ""},
+		{"pointer receiver", "simd/archsimd", "*X86Features", "AVX2() bool", ""},
+		{"other query", "simd/archsimd", "X86Features", "AVX() bool", ""},
+		{"argument", "simd/archsimd", "X86Features", "AVX2(int) bool", ""},
+		{"result", "simd/archsimd", "X86Features", "AVX2() int", ""},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			fs := token.NewFileSet()
+			file, err := parser.ParseFile(fs, "cpu.go", "package archsimd; type X86Features struct{}; type OtherFeatures struct{}; func ("+tc.recv+") "+tc.signature, 0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			pkg, err := new(types.Config).Check(tc.path, fs, []*ast.File{file}, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			prog := ssa.NewProgram(fs, 0)
+			prog.CreatePackage(pkg, nil, nil, true)
+			named := pkg.Scope().Lookup(strings.TrimPrefix(tc.recv, "*")).Type().(*types.Named)
+			if got := simdCPUQuery(prog.FuncValue(named.Method(0))); got != tc.want {
+				t.Fatalf("query = %q, want %q", got, tc.want)
+			}
+		})
+	}
+}
+
 func TestSIMDOperationIdentity(t *testing.T) {
 	const source = `package archsimd
  type v128 struct { _ [0]func() }

dev/test_native_simd.sh

@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+
+: "${LLGO:=llgo}"
+export GOEXPERIMENT=simd
+
+if [[ "$(go env GOARCH)" != amd64 ]]; then
+  "$LLGO" test -O0 -count=1 -timeout=2m ./test/simd/...
+  "$LLGO" test -O2 -count=1 -timeout=2m ./test/simd/...
+  exit
+fi
+
+export GOAMD64=v1
+if [[ -n "${LLGO_SIMD_ARTIFACT_DIR:-}" ]]; then
+  output=$LLGO_SIMD_ARTIFACT_DIR
+  mkdir -p "$output"
+else
+  output=$(mktemp -d)
+  trap 'rm -rf "$output"' EXIT
+fi
+
+for profile in O0-off O2-off O2-thin O2-full; do
+  opt=${profile%%-*}
+  lto=${profile#*-}
+  flags=("-$opt")
+  if [[ "$lto" != off ]]; then
+    flags+=("-lto=$lto")
+  fi
+  echo "SIMD: amd64 $profile"
+  binary="$output/simd-$profile.exe"
+  "$LLGO" test "${flags[@]}" -c -o "$binary" ./test/simd
+  "$binary" -test.v -test.count=1 -test.timeout=2m
+  for debug in cpu.avx2=off cpu.avx=off cpu.fma=off cpu.all=off; do
+    echo "SIMD FMV: $profile GODEBUG=$debug"
+    GODEBUG="$debug" "$binary" -test.v -test.count=1 -test.timeout=2m -test.run='^TestSIMDFMV'
+  done
+done

go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/mattn/go-tty v0.0.8
 	github.com/qiniu/x v1.19.2
 	github.com/xgo-dev/llgo/runtime v0.0.0-00010101000000-000000000000
-	github.com/xgo-dev/llvm v0.10.0
+	github.com/xgo-dev/llvm v0.10.1
 	github.com/xgo-dev/plan9asm v0.6.2
 	go.bug.st/serial v1.6.4
 	go.yaml.in/yaml/v3 v3.0.5

go.sum

@@ -24,6 +24,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/xgo-dev/llvm v0.10.0 h1:o27kypAGI7DrLFckrsUVjWChXqHUBDAFtZbwMiDFs50=
 github.com/xgo-dev/llvm v0.10.0/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M=
+github.com/xgo-dev/llvm v0.10.1 h1:GZeoapgSKC1Qgfy+xQmlIAKg0Suvrpg4U3Y3xticjqw=
+github.com/xgo-dev/llvm v0.10.1/go.mod h1:42vav2/cI5BAIcL543DZSMO9do8/aCK2z7JERH+AE+M=
 github.com/xgo-dev/plan9asm v0.6.2 h1:9mV7u9stDT2V3wZP+ywRP/wbWmtQwf5bb6zym5Unyww=
 github.com/xgo-dev/plan9asm v0.6.2/go.mod h1:UTpcZz3aQXSWYruhfzOi8aK4QZTEFPIvB61YW6jHT4M=
 go.bug.st/serial v1.6.4 h1:7FmqNPgVp3pu2Jz5PoPtbZ9jJO5gnEnZIvnI1lzve8A=

internal/build/build.go

@@ -56,6 +56,7 @@ import (
 	"github.com/xgo-dev/llgo/internal/goarch"
 	"github.com/xgo-dev/llgo/internal/goembed"
 	"github.com/xgo-dev/llgo/internal/header"
+	"github.com/xgo-dev/llgo/internal/llvmfmv"
 	"github.com/xgo-dev/llgo/internal/lto"
 	"github.com/xgo-dev/llgo/internal/meta"
 	"github.com/xgo-dev/llgo/internal/mockable"
@@ -3285,6 +3286,7 @@ func lowerMainCExportModule(ctx *context, pkg llssa.Package, exports []cExport)
 	if !lowerMainCExportAggregates(ctx.prog, mod, exports) {
 		return false, nil
 	}
+	ctx.cTransformer.LowerWindowsVectorParams(mod)
 	ctx.cTransformer.TransformModule(pkg.Path(), mod)
 	if ctx.buildConf.Goarch != "wasm" {
 		return true, nil
@@ -3330,6 +3332,10 @@ func compilePackageModule(ctx *context, aPkg *aPackage, externs []string, verbos
 	ret := aPkg.LPkg
 
 	ctx.cTransformer.SetSkipFuncs(cabiSkipFuncsForPlan9Asm(ctx, pkgPath, ret.Module()))
+	ctx.cTransformer.LowerWindowsVectorParams(ret.Module())
+	if err := llvmfmv.Run(ret.Module(), ctx.buildConf.GOAMD64); err != nil {
+		return fmt.Errorf("%s: %w", pkgPath, err)
+	}
 	lowerLargeAggregates(ctx.prog, ret.Module())
 	ctx.cTransformer.TransformModule(ret.Path(), ret.Module())
 	ctx.cTransformer.SetSkipFuncs(nil)

internal/build/large_roots_test.go

@@ -1,6 +1,7 @@
 package build
 
 import (
+	"go/token"
 	"go/types"
 	"strings"
 	"testing"
@@ -10,6 +11,44 @@ import (
 	"github.com/xgo-dev/llvm"
 )
 
+func TestLowerMainCExportModuleWindowsVectors(t *testing.T) {
+	llvm.InitializeAllTargets()
+	llvm.InitializeAllTargetMCs()
+	llvm.InitializeAllTargetInfos()
+	target := &llssa.Target{GOOS: "windows", GOARCH: "amd64"}
+	prog := llssa.NewProgram(target)
+	defer prog.Dispose()
+	pkg := prog.NewPackage("exportprobe", "exportprobe")
+	rawPkg := types.NewPackage("simd/archsimd", "archsimd")
+	marker := types.NewNamed(types.NewTypeName(token.NoPos, rawPkg, "v128", nil), types.NewStruct(nil, nil), nil)
+	vector := types.NewNamed(types.NewTypeName(token.NoPos, rawPkg, "Float32x4", nil), types.NewStruct([]*types.Var{
+		types.NewField(token.NoPos, rawPkg, "tag", marker, false),
+		types.NewField(token.NoPos, rawPkg, "vals", types.NewArray(types.Typ[types.Float32], 4), false),
+	}, nil), nil)
+	exports := []cExport{{goName: "main.Exported", cName: "Exported", sig: newSignature([]types.Type{vector}, []types.Type{vector})}}
+	defineCExportWrappers(pkg, exports, nil)
+	ctx := &context{
+		prog: prog, buildConf: &Config{Goos: "windows", Goarch: "amd64"},
+		cTransformer: cabi.NewTransformer(prog, target.Spec().Triple, "", true),
+	}
+	if changed, err := lowerMainCExportModule(ctx, pkg, exports); err != nil || !changed {
+		t.Fatalf("C export lowering = (%t, %v)", changed, err)
+	}
+	mod := pkg.Module()
+	for _, name := range []string{"Exported", "main.Exported"} {
+		fn := mod.NamedFunction(name)
+		if fn.Param(0).Type().TypeKind() != llvm.PointerTypeKind || fn.GlobalValueType().ReturnType().TypeKind() != llvm.VectorTypeKind {
+			t.Fatalf("Win64 export must use an indirect vector parameter and direct vector return:\n%s", fn.String())
+		}
+	}
+	if !strings.Contains(mod.NamedFunction("Exported").String(), "@main.Exported(ptr ") {
+		t.Fatalf("export call does not match the lowered implementation:\n%s", mod.String())
+	}
+	if err := llvm.VerifyModule(mod, llvm.ReturnStatusAction); err != nil {
+		t.Fatal(err)
+	}
+}
+
 func TestLowerLargeAggregatesGCPolicy(t *testing.T) {
 	llvm.InitializeAllTargets()
 	llvm.InitializeAllTargetMCs()