Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
fail-fast: false
matrix:
go-version: [1.25.x, 1.24.x, 1.22.x, 1.21.x]
platform: [ubuntu-latest, windows-latest]
platform: [ubuntu-latest, windows-latest, macos-15-intel, macos-15]
#platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
steps:
Expand Down Expand Up @@ -51,6 +51,12 @@ jobs:
# install goimports
go install golang.org/x/tools/cmd/goimports@latest

- name: Install macOS packages
if: startsWith(matrix.platform, 'macos-')
run: |
python3 -m pip install -U pybindgen
go install golang.org/x/tools/cmd/goimports@latest

- name: Install Windows packages
if: matrix.platform == 'windows-latest'
run: |
Expand All @@ -69,6 +75,15 @@ jobs:
run: |
make test

- name: Build-macOS
if: startsWith(matrix.platform, 'macos-')
run: |
make
- name: Test macOS
if: startsWith(matrix.platform, 'macos-')
run: |
make test

- name: Build-Windows
if: matrix.platform == 'windows-latest'
run: |
Expand Down
1 change: 1 addition & 0 deletions SUPPORT_MATRIX.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ _examples/consts | yes
_examples/cstrings | yes
_examples/empty | yes
_examples/funcs | yes
_examples/gilstring | yes
_examples/gobytes | yes
_examples/gopygc | yes
_examples/gostrings | yes
Expand Down
2 changes: 1 addition & 1 deletion _examples/cgo/cgo.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ package cgo
//#include <string.h>
//#include <stdlib.h>
//const char* cpkg_sprintf(const char *str) {
// char *o = (char*)malloc(strlen(str));
// char *o = (char*)malloc(strlen(str) + 1);
// sprintf(o, "%s", str);
// return o;
//}
Expand Down
14 changes: 14 additions & 0 deletions _examples/gilstring/gilstring.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// Copyright 2026 The go-python Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package gilstring is a regression test for the multi-runtime crash (issue #370).
// It mirrors the exact reproduction from the issue report: a string-returning
// function called alongside an integer function from a second extension in the
// same Python process, which triggers crashes under repeated calls.
package gilstring

import "fmt"

// Hello returns a greeting string, mirroring hi.Hello from the issue report.
func Hello(s string) string { return fmt.Sprintf("Hello, %s!", s) }
18 changes: 18 additions & 0 deletions _examples/gilstring/test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# Copyright 2026 The go-python Authors. All rights reserved.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

## py2/py3 compat
from __future__ import print_function

# Regression test for multi-runtime crash (issue #370).
# Exact reproduction from the issue report: two separately-built gopy
# extensions loaded in the same process, with calls interleaved in a loop.
from gilstring.gilstring import Hello
from simple.simple import Add

for _ in range(5000):
Add(2, 2)
Hello('hi')

print("OK")
59 changes: 59 additions & 0 deletions bind/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,27 @@ static inline void gopy_err_handle() {
PyErr_Print();
}
}
// _gopy_clear_go_tls clears the Go goroutine pointer from this thread's TLS.
// When multiple gopy extensions share a process, each has its own Go runtime
// but all runtimes use the same TLS slot for the current goroutine pointer
// (GS:0x30 on darwin/amd64, FS:-8 on linux/amd64). After one extension's
// init(), TLS is left pointing to that runtime's g0. If another extension's
// CGo entry-point reads TLS and finds a non-nil goroutine, it takes the fast
// path (no needm()) and runs with the wrong M/P/mcache -- corrupting the heap.
// Clearing the slot before each CGo entry forces needm() to run, which
// establishes the correct per-extension context (issue #370).
static void _gopy_clear_go_tls(void) {
#if defined(__x86_64__) && defined(__APPLE__)
__asm__ volatile("movq $0, %%%%gs:0x30" ::: "memory");
#elif defined(__x86_64__) && defined(__linux__)
__asm__ volatile("movq $0, %%%%fs:-8" ::: "memory");
#endif
}
%[8]s
*/
import "C"
import (
"runtime"
"github.com/go-python/gopy/gopyh" // handler
%[6]s
)
Expand Down Expand Up @@ -132,6 +149,15 @@ func NumHandles() int {
return gopyh.NumHandles()
}

// RunGC runs the Go garbage collector. gopy registers this as a Python
// gc.callbacks handler so it fires automatically after each Python GC cycle,
// keeping Go-heap objects freed via DecRef actually collected without any
// user intervention.
//export RunGC
func RunGC() {
runtime.GC()
}

// boolGoToPy converts a Go bool to python-compatible C.char
func boolGoToPy(b bool) C.char {
if b {
Expand Down Expand Up @@ -259,6 +285,8 @@ mod.add_function('GoPyInit', None, [])
mod.add_function('DecRef', None, [param('int64_t', 'handle')])
mod.add_function('IncRef', None, [param('int64_t', 'handle')])
mod.add_function('NumHandles', retval('int'), [])
mod.add_function('RunGC', None, [])
mod.add_function('_gopy_clear_go_tls', None, [])
`

// appended to imports in py wrap preamble as key for adding at end
Expand All @@ -281,8 +309,39 @@ except ImportError:
cwd = os.getcwd()
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
os.chdir(currentdir)
# When multiple gopy extensions coexist in one Python process each carries its own
# independent Go runtime. The Go extension is loaded without RTLD_GLOBAL below, and
# _gopy_clear_go_tls() is called before each CGo entry to force needm() to run, which
# establishes the correct per-extension M/P/mcache context (issue #370).
# Also load the extension without RTLD_GLOBAL so that Go runtime symbols stay
# local to each .so — belt-and-suspenders on platforms where RTLD_GLOBAL is the
# Python default (e.g. some Linux builds).
if hasattr(sys, 'getdlopenflags'):
try:
import ctypes as _gopy_ctypes
_gopy_saved_flags = sys.getdlopenflags()
sys.setdlopenflags(_gopy_saved_flags & ~getattr(_gopy_ctypes, 'RTLD_GLOBAL', 0))
except Exception:
_gopy_saved_flags = None
else:
_gopy_saved_flags = None
%[6]s
if _gopy_saved_flags is not None:
sys.setdlopenflags(_gopy_saved_flags)
os.chdir(cwd)
# Run Go's GC whenever Python's GC runs so that Go-heap objects whose handles
# were released via DecRef are promptly collected. Without this, Go memory
# can accumulate between Python gc.collect() calls because Python GC only
# frees the Python wrapper; the underlying Go allocation is not reclaimed
# until Go's own GC fires.
try:
import gc as _gopy_gc
def _gopy_gc_cb(phase, info):
if phase == 'stop':
_%[1]s.RunGC()
_gopy_gc.callbacks.append(_gopy_gc_cb)
except Exception:
pass

# to use this code in your end-user python file, import it as follows:
# from %[1]s import %[3]s
Expand Down
9 changes: 6 additions & 3 deletions bind/gen_func.go
Original file line number Diff line number Diff line change
Expand Up @@ -261,10 +261,8 @@ func (g *pyGen) genFuncBody(sym *symbol, fsym *Func) {
}
}

// release GIL
g.gofile.Printf("_saved_thread := C.PyEval_SaveThread()\n")
if !rvIsErr && nres != 2 {
// reacquire GIL after return
g.gofile.Printf("defer C.PyEval_RestoreThread(_saved_thread)\n")
}

Expand Down Expand Up @@ -338,6 +336,12 @@ if __err != nil {
}
}

// Clear the Go TLS goroutine slot before the CGo entry point so that
// Go's needm() runs and establishes the correct per-extension context.
// Without this, two extensions sharing the same process can corrupt
// each other's heap via TLS collision (issue #370).
g.pywrap.Printf("_%s._gopy_clear_go_tls()\n", pkgname)

// pywrap output
mnm := fsym.ID()
if isMethod {
Expand Down Expand Up @@ -415,7 +419,6 @@ if __err != nil {

if rvIsErr || nres == 2 {
g.gofile.Printf("\n")
// reacquire GIL
g.gofile.Printf("C.PyEval_RestoreThread(_saved_thread)\n")

g.gofile.Printf("if __err != nil {\n")
Expand Down
16 changes: 12 additions & 4 deletions bind/symbols.go
Original file line number Diff line number Diff line change
Expand Up @@ -1083,24 +1083,32 @@ func (sym *symtab) addSignatureType(pkg *types.Package, obj types.Object, t type

py2g := fmt.Sprintf("%s { ", nsig)

py2g += "_gstate := C.PyGILState_Ensure()\n"

// TODO: use strings.Builder
if rets.Len() == 0 {
py2g += "if C.PyCallable_Check(_fun_arg) == 0 { return }\n"
py2g += "if C.PyCallable_Check(_fun_arg) == 0 {\n"
py2g += "C.PyGILState_Release(_gstate)\n" // Release GIL
py2g += "return\n"
py2g += "}\n"
} else {
zstr, err := sym.ZeroToGo(ret.Type(), rsym)
if err != nil {
return err
}
py2g += fmt.Sprintf("if C.PyCallable_Check(_fun_arg) == 0 { return %s }\n", zstr)
py2g += "if C.PyCallable_Check(_fun_arg) == 0 {\n"
py2g += "C.PyGILState_Release(_gstate)\n" // Release GIL
py2g += fmt.Sprintf("return %s\n", zstr)
py2g += "}\n"
}
py2g += "_gstate := C.PyGILState_Ensure()\n"

if nargs > 0 {
bstr, err := sym.buildTuple(args, "_fcargs", "_fun_arg")
if err != nil {
return err
}
py2g += bstr + retstr
py2g += fmt.Sprintf("C.PyObject_CallObject(_fun_arg, _fcargs)\n")
py2g += "C.PyObject_CallObject(_fun_arg, _fcargs)\n"
py2g += "C.gopy_decref(_fcargs)\n"
} else {
// TODO: methods not supported for no-args case -- requires self arg..
Expand Down
53 changes: 45 additions & 8 deletions cmd_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,30 +181,67 @@ func runBuild(mode bind.BuildMode, cfg *BuildCfg) error {

// build the go shared library upfront to generate the header
// needed by our generated cpython code
args := []string{"build", "-mod=mod", "-buildmode=c-shared"}
firstArgs := []string{"build", "-mod=mod", "-buildmode=c-shared"}
if cfg.BuildTags != "" {
args = append(args, "-tags", cfg.BuildTags)
firstArgs = append(firstArgs, "-tags", cfg.BuildTags)
}
if !cfg.Symbols {
// These flags will omit the various symbol tables, thereby
// reducing the final size of the binary. From https://golang.org/cmd/link/
// -s Omit the symbol table and debug information
// -w Omit the DWARF symbol table
args = append(args, "-ldflags=-s -w")
firstArgs = append(firstArgs, "-ldflags=-s -w")
}
args = append(args, "-o", buildLib, ".")
fmt.Printf("go %v\n", strings.Join(args, " "))
cmd = exec.Command("go", args...)
firstArgs = append(firstArgs, "-o", buildLib, ".")
fmt.Printf("go %v\n", strings.Join(firstArgs, " "))
cmd = exec.Command("go", firstArgs...)
cmdout, err = cmd.CombinedOutput()
if err != nil {
fmt.Printf("cmd had error: %v output:\n%v\n", err, string(cmdout))
return err
}
// update the output name to the one with the ABI extension
args[len(args)-2] = modlib
// we don't need this initial lib because we are going to relink
os.Remove(buildLib)

// Build the final extension with symbol-visibility restriction so that
// Go runtime globals are not placed in the global dynamic-linker
// namespace. Two independently-loaded Go runtimes sharing those globals
// via RTLD_GLOBAL interposition corrupt each other's GC state (#370).
// This applies only to the second build, which is where PyInit__<name>
// exists and where the exported-symbols list is valid.
finalArgs := []string{"build", "-mod=mod", "-buildmode=c-shared"}
if cfg.BuildTags != "" {
finalArgs = append(finalArgs, "-tags", cfg.BuildTags)
}
var finalLdFlags []string
if !cfg.Symbols {
finalLdFlags = append(finalLdFlags, "-s", "-w")
}
switch runtime.GOOS {
case "darwin":
ef, ferr := os.CreateTemp("", "gopy-exports-*.txt")
if ferr == nil {
fmt.Fprintf(ef, "_PyInit__%s\n", cfg.Name)
ef.Close()
defer os.Remove(ef.Name())
finalLdFlags = append(finalLdFlags, "-extldflags=-Wl,-exported_symbols_list,"+ef.Name())
}
case "linux":
ef, ferr := os.CreateTemp("", "gopy-exports-*.map")
if ferr == nil {
fmt.Fprintf(ef, "{ global: PyInit__%s; local: *; };\n", cfg.Name)
ef.Close()
defer os.Remove(ef.Name())
finalLdFlags = append(finalLdFlags, "-extldflags=-Wl,--version-script="+ef.Name())
}
}
if len(finalLdFlags) > 0 {
finalArgs = append(finalArgs, "-ldflags="+strings.Join(finalLdFlags, " "))
}
finalArgs = append(finalArgs, "-o", modlib, ".")
// args is still used below for the CGO env build; point it at finalArgs.
args := finalArgs

// generate c code
fmt.Printf("%v build.py\n", cfg.VM)
cmd = exec.Command(cfg.VM, "build.py")
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,13 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4=
golang.org/x/mod v0.22.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/telemetry v0.0.0-20240521205824-bda55230c457/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0=
golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE=
golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588=
Loading
Loading