diff --git a/.github/workflows/llgo.yml b/.github/workflows/llgo.yml index 7768bb0885..8255aeae1a 100644 --- a/.github/workflows/llgo.yml +++ b/.github/workflows/llgo.yml @@ -571,6 +571,11 @@ jobs: iwasm --stack-size=819200000 --heap-size=800000000 hello.wasm + - name: Set up wasm-tools for EH fixtures + uses: bytecodealliance/actions/wasm-tools/setup@v1 + with: + version: "1.243.0" + - name: Test WASI pthread backend and threaded GC (WAMR) shell: bash run: python3 dev/test_wasm_wasi_threads.py diff --git a/dev/build_iwasm.sh b/dev/build_iwasm.sh index 9da307f5ca..9ef2858a11 100755 --- a/dev/build_iwasm.sh +++ b/dev/build_iwasm.sh @@ -27,6 +27,8 @@ IWASM_BUILD_ID=$( printf '%s\n' "${WAMR_VERSION}" "$(uname -s)" "$(uname -m)" \ "${LLGO_WINDOWS_ABI:-}" "${MINGW_PREFIX:-}" "${CC:-}" "${CXX:-}" \ "$(git hash-object "${SCRIPT_DIR}/build_iwasm.sh")" \ + "$(git hash-object "${SCRIPT_DIR}/patches/wamr-2.4.5-threaded-eh.patch")" \ + "$(git hash-object "${SCRIPT_DIR}/patches/wamr-2.4.5-posix-signal.patch")" \ "$(git hash-object "${SCRIPT_DIR}/patches/wamr-2.4.5-mingw.patch")" \ | git hash-object --stdin ) @@ -55,6 +57,13 @@ cd "${TEMP_DIR}" echo "Cloning wasm-micro-runtime ${WAMR_VERSION}..." git clone --branch "${WAMR_VERSION}" --depth 1 https://github.com/wasm-micro-runtime/wasm-micro-runtime.git +# Keep exception propagation within one interpreter thread until it escapes +# the Wasm invocation. Otherwise a caught longjmp can terminate sibling Gs. +git -C wasm-micro-runtime apply \ + "${SCRIPT_DIR}/patches/wamr-2.4.5-threaded-eh.patch" +git -C wasm-micro-runtime apply \ + "${SCRIPT_DIR}/patches/wamr-2.4.5-posix-signal.patch" + CMAKE_GENERATOR_ARGS=() case "$(uname -s)" in Darwin) @@ -145,7 +154,7 @@ cmake "${CMAKE_GENERATOR_ARGS[@]}" \ -D WAMR_BUILD_SHARED_MEMORY=1 \ -D WAMR_BUILD_LIB_WASI_THREADS=1 \ -D WAMR_BUILD_LIB_PTHREAD=1 \ - -D CMAKE_BUILD_TYPE=Debug \ + -D CMAKE_BUILD_TYPE=Release \ -D WAMR_BUILD_DEBUG_INTERP=0 \ .. diff --git a/dev/patches/wamr-2.4.5-posix-signal.patch b/dev/patches/wamr-2.4.5-posix-signal.patch new file mode 100644 index 0000000000..7d01526739 --- /dev/null +++ b/dev/patches/wamr-2.4.5-posix-signal.patch @@ -0,0 +1,72 @@ +Backport of https://github.com/wasm-micro-runtime/wasm-micro-runtime/pull/5119 +Install process-wide signal handlers once while retaining per-thread stacks. + +diff --git a/core/shared/platform/common/posix/posix_thread.c b/core/shared/platform/common/posix/posix_thread.c +index 80b7d654..f9350a21 100644 +--- a/core/shared/platform/common/posix/posix_thread.c ++++ b/core/shared/platform/common/posix/posix_thread.c +@@ -596,6 +596,8 @@ mask_signals(int how) + + static struct sigaction prev_sig_act_SIGSEGV; + static struct sigaction prev_sig_act_SIGBUS; ++static pthread_once_t signal_handler_once = PTHREAD_ONCE_INIT; ++static int signal_handler_install_result = -1; + + /* + * ASAN is not designed to work with custom stack unwind or other low-level +@@ -654,10 +656,30 @@ signal_callback(int sig_num, siginfo_t *sig_info, void *sig_ucontext) + } + } + ++static void ++install_signal_handler_once(void) ++{ ++ struct sigaction sig_act; ++ ++ memset(&sig_act, 0, sizeof(sig_act)); ++ sig_act.sa_sigaction = signal_callback; ++ sig_act.sa_flags = SA_SIGINFO | SA_NODEFER; ++#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 ++ sig_act.sa_flags |= SA_ONSTACK; ++#endif ++ sigemptyset(&sig_act.sa_mask); ++ if (sigaction(SIGSEGV, &sig_act, &prev_sig_act_SIGSEGV) != 0) ++ return; ++ if (sigaction(SIGBUS, &sig_act, &prev_sig_act_SIGBUS) != 0) { ++ sigaction(SIGSEGV, &prev_sig_act_SIGSEGV, NULL); ++ return; ++ } ++ signal_handler_install_result = 0; ++} ++ + int + os_thread_signal_init(os_signal_handler handler) + { +- struct sigaction sig_act; + #if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 + stack_t sigalt_stack_info; + uint32 map_size = SIG_ALT_STACK_SIZE; +@@ -693,18 +715,11 @@ os_thread_signal_init(os_signal_handler handler) + } + #endif + +- memset(&prev_sig_act_SIGSEGV, 0, sizeof(struct sigaction)); +- memset(&prev_sig_act_SIGBUS, 0, sizeof(struct sigaction)); +- +- /* Install signal handler */ +- sig_act.sa_sigaction = signal_callback; +- sig_act.sa_flags = SA_SIGINFO | SA_NODEFER; +-#if WASM_DISABLE_STACK_HW_BOUND_CHECK == 0 +- sig_act.sa_flags |= SA_ONSTACK; +-#endif +- sigemptyset(&sig_act.sa_mask); +- if (sigaction(SIGSEGV, &sig_act, &prev_sig_act_SIGSEGV) != 0 +- || sigaction(SIGBUS, &sig_act, &prev_sig_act_SIGBUS) != 0) { ++ /* sigaction is process-wide, although the alternate stack and handler ++ * context above are per-thread. Reinstalling here would make the previous ++ * action point back to signal_callback after the first worker starts. */ ++ if (pthread_once(&signal_handler_once, install_signal_handler_once) != 0 ++ || signal_handler_install_result != 0) { + os_printf("Failed to register signal handler\n"); + goto fail3; + } diff --git a/dev/patches/wamr-2.4.5-threaded-eh.patch b/dev/patches/wamr-2.4.5-threaded-eh.patch new file mode 100644 index 0000000000..f58ba196ac --- /dev/null +++ b/dev/patches/wamr-2.4.5-threaded-eh.patch @@ -0,0 +1,28 @@ +WAMR 2.4.5: do not broadcast catchable intra-Wasm exceptions to other threads. +LLGo regression: internal/build/testdata/wasm-wasi-goexit-defer. +The terminal uncaught-exception path still publishes a cluster-wide trap. + +diff --git a/core/iwasm/interpreter/wasm_interp_classic.c b/core/iwasm/interpreter/wasm_interp_classic.c +index 9f7e03e2..61fccd76 100644 +--- a/core/iwasm/interpreter/wasm_interp_classic.c ++++ b/core/iwasm/interpreter/wasm_interp_classic.c +@@ -1933,6 +1933,19 @@ wasm_interp_call_func_bytecode(WASMModuleInstance *module, + *((int32 *)(prev_frame->sp)) = exception_tag_index; + prev_frame->sp++; + ++ /* Keep a catchable exception inside this Wasm ++ * invocation. Publishing it through ++ * wasm_set_exception would cancel every thread in ++ * the cluster before the caller can catch it. */ ++ if (prev_frame->ip) { ++ FREE_FRAME(exec_env, frame); ++ wasm_exec_env_set_cur_frame(exec_env, ++ prev_frame); ++ RECOVER_CONTEXT(prev_frame); ++ exception_tag_index = POP_I32(); ++ goto find_a_catch_handler; ++ } ++ + /* mark frame as raised exception */ + wasm_set_exception(module, + "uncaught wasm exception"); diff --git a/dev/test_wasm_wasi_threads.py b/dev/test_wasm_wasi_threads.py index a077012938..60837ba7df 100644 --- a/dev/test_wasm_wasi_threads.py +++ b/dev/test_wasm_wasi_threads.py @@ -7,6 +7,7 @@ import shutil import subprocess import tempfile +import time ROOT = pathlib.Path(__file__).resolve().parent.parent @@ -15,7 +16,7 @@ def run_probe(env, directory, name, fixture, tags, marker, timeout, - max_threads=8, expected_exit=0): + max_threads=8, expected_exit=0, args=(), runs=1): module = pathlib.Path(directory) / f"{name}.wasm" command = [LLGO, "build", "-target", "wasi"] if tags: @@ -27,21 +28,23 @@ def run_probe(env, directory, name, fixture, tags, marker, timeout, env=env, timeout=180, ) - result = subprocess.run( - [IWASM, f"--max-threads={max_threads}", "--stack-size=1048576", - "--heap-size=0", "--dir=" + str(ROOT), "--dir=/tmp", str(module)], - capture_output=True, - text=True, - timeout=timeout, - ) - print(result.stdout, end="") - print(result.stderr, end="") allowed_exits = (expected_exit,) if isinstance(expected_exit, int) else expected_exit - if result.returncode not in allowed_exits or ( - marker not in result.stdout.splitlines() - and marker not in result.stderr.splitlines() - ): - raise SystemExit(f"WAMR {name} probe failed with exit code {result.returncode}") + for attempt in range(runs): + result = subprocess.run( + [IWASM, f"--max-threads={max_threads}", "--stack-size=1048576", + "--heap-size=0", "--dir=" + str(ROOT), "--dir=/tmp", str(module), *args], + capture_output=True, + text=True, + timeout=timeout, + ) + print(result.stdout, end="") + print(result.stderr, end="") + markers = (marker,) if isinstance(marker, str) else marker + lines = result.stdout.splitlines() + result.stderr.splitlines() + if result.returncode not in allowed_exits or any(m not in lines for m in markers): + raise SystemExit( + f"WAMR {name} probe {attempt + 1}/{runs} failed " + f"with exit code {result.returncode}") def run_llgo(env, args, marker, timeout=180): @@ -94,6 +97,33 @@ def main(): # WAMR can translate the terminal Wasm exception to process status 1 # instead of preserving the guest's status 2. Both are nonzero exits. deadlock_exits = (1, 2) + # LLVM lowers both Go defer/Goexit and C setjmp/longjmp through legacy + # Wasm EH. A caught exception must not terminate unrelated pthreads. + for tags in ("nogc", ""): + suffix = tags or "gc" + run_probe(env, directory, f"deferred-goexit-{suffix}", + "wasm-wasi-goexit-defer", tags, + "wasi worker defer ok", 30, runs=10) + for mode in ("init", "main"): + run_probe(env, directory, f"{mode}-defer-{suffix}", + "wasm-wasi-goexit-defer", tags, + (f"wasi {mode} defer ok", + "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"), + 30, expected_exit=deadlock_exits, args=(mode,)) + run_probe(env, directory, f"uncaught-{suffix}", + "wasm-wasi-goexit-defer", tags, + "panic: wasi uncaught sentinel", 30, + expected_exit=deadlock_exits, args=("uncaught",)) + # A raw exception escaping the Wasm entry point must still fail. Go's + # unrecovered panic exits explicitly, so it cannot test this boundary. + uncaught = pathlib.Path(directory) / "uncaught-eh.wasm" + subprocess.run([os.environ.get("WASM_TOOLS", "wasm-tools"), "parse", + str(ROOT / "internal/build/testdata/wasm-wasi-goexit-defer/uncaught.wat"), + "-o", str(uncaught)], check=True, timeout=30) + result = subprocess.run([IWASM, str(uncaught)], capture_output=True, + text=True, timeout=30) + if result.returncode == 0 or "uncaught wasm exception" not in result.stdout + result.stderr: + raise SystemExit(f"WAMR swallowed an escaping exception: {result}") run_probe(env, directory, "main-goexit", "wasm-wasi-main-goexit", "nogc", "fatal error: no goroutines (main called runtime.Goexit) - deadlock!", 30, expected_exit=deadlock_exits) @@ -122,6 +152,33 @@ def main(): "wasi threaded filesystem ok") run_llgo(env, ["test", "-target", "wasi", "-emulator", str(ROOT / "test/std/errors")], "PASS") + # Compiling test/go on a cold CI runner and running the GC race are + # separate budgets. Verbose, inherited output identifies a slow test + # immediately instead of discarding it when subprocess.run times out. + module = pathlib.Path(directory) / "runtime-gc-tests.wasm" + started = time.monotonic() + subprocess.run([LLGO, "test", "-c", "-target", "wasi", "-o", str(module), + str(ROOT / "test/go")], env=env, check=True, timeout=300) + print(f"WAMR GC test compilation: {time.monotonic() - started:.2f}s", flush=True) + # The two startup shapes each race 20 goroutines against continuous + # full GC. The classic interpreter takes over two minutes per shape + # on CI. Give each an independent runtime/deadline, and keep finalizer, + # callback GC and first-use symbol lookup in a third invocation. All + # cases and repetition counts remain enabled; a stalled case still + # fails within 300 seconds with its last active test visible. + runtime_cases = ( + ("finalizers/callback/symbols", + "^(TestRuntimeSetFinalizer.*|TestReflectMakeFuncGoroutineGC|TestRuntimeFuncInfoConcurrentFirstUse)$"), + ("startup-pointer", "^TestReflectMakeFuncGoroutineStartup$/^pointer_argument$"), + ("startup-zero", "^TestReflectMakeFuncGoroutineStartup$/^zero_arguments$"), + ) + for name, pattern in runtime_cases: + started = time.monotonic() + subprocess.run([IWASM, "--max-threads=128", "--stack-size=1048576", "--heap-size=0", + "--dir=" + str(ROOT), "--dir=/tmp", str(module), "-test.v", + "-test.run=" + pattern], + env=env, check=True, timeout=300) + print(f"WAMR GC test {name}: {time.monotonic() - started:.2f}s", flush=True) run_llgo(env, ["test", "-target", "wasi", "-emulator", "-run", "^TestPoolAfterGC$", str(ROOT / "test/std/sync")], "PASS", timeout=300) diff --git a/dev/wasm-eh-comparison.md b/dev/wasm-eh-comparison.md index 87de977088..b6545e5d29 100644 --- a/dev/wasm-eh-comparison.md +++ b/dev/wasm-eh-comparison.md @@ -69,3 +69,33 @@ provided panic/recover, Asyncify suspension, Go/C/JS callbacks, final DWARF, and the chosen runtime all pass together. [WAMR's documented EH support](https://github.com/bytecodealliance/wasm-micro-runtime/blob/main/doc/build_wamr.md) is currently limited to legacy EH in its classic interpreter, so the browser comparison alone does not change the W32/WAMR execution contract. + +## Threaded WAMR regression + +`python3 dev/test_wasm_wasi_threads.py` also tests the supported W32 legacy EH +path with the WAMR built by `dev/build_iwasm.sh`. It repeats cross-function +panic/recover, C `setjmp`/`longjmp`, and deferred worker `Goexit` with GC on and +off. Main/init `Goexit` must execute the defer and report deadlock; an +unrecovered worker panic and a raw Wasm exception escaping `_start` must fail. + +WAMR 2.4.5 previously called `wasm_set_exception` while transferring a caught +exception to its Wasm caller. With threads enabled, that publishes a +cluster-wide termination signal before the caller can catch the exception. +Sibling threads can then exit early or leave a channel waiter hung. The local +interpreter patch unwinds directly to a Wasm caller and preserves the terminal +exception path when the exception escapes to the native invocation boundary. +The POSIX signal-handler backport from WAMR #5119 is applied separately. + +On macOS arm64, the same deferred-Goexit artifact passed 40/50 runs with stock +WAMR 2.4.5 (six hangs and four premature successful exits), 44/50 with the +signal-handler fix alone on WAMR main, and 100/100 with both fixes on 2.4.5. +The threaded acceptance suite and the expanded GC/nogc EH probes passed with +both fixes. These finite runs establish regression coverage, not a guarantee +that every WAMR threading issue is resolved. + +The browser comparison was rerun on 2026-09-29 with the pinned LLGo Binaryen +`llgo-v132.3`: all six C++ encoding/optimization variants passed in Node and +Chrome, as did the Go baseline and the Go/C++ catch-status wrappers at O0/O2. +This keeps the supported boundary above: browser encoding is unchanged, +and WAMR uses legacy EH. Whole-module exnref across Go/Asyncify remains outside +the supported contract. diff --git a/dev/wasm-wasi-validation.md b/dev/wasm-wasi-validation.md new file mode 100644 index 0000000000..56d664c68d --- /dev/null +++ b/dev/wasm-wasi-validation.md @@ -0,0 +1,75 @@ +# WAMR threads and exception validation + +The W32 backend keeps one goroutine per pthread. The following checks qualify +that backend; they do not introduce an M:N scheduler on native or WASI targets. + +## Fixed failure modes + +- WAMR 2.4.5's classic interpreter briefly broadcast thread termination while + propagating a catchable exception to a Wasm caller. The interpreter patch + keeps that propagation local. The original deferred-Goexit artifact passed + 40/50 runs on stock WAMR and 100/100 with the patch. Escaping exceptions still + fail the invocation. Encoding comparisons and the supported Go/C++ boundary + are recorded in [wasm-eh-comparison.md](wasm-eh-comparison.md). +- A pthread condition wait can block while reacquiring a mutex held by a Go + thread already stopped for collection. Merely polling the condition every + 20 ms did not resolve that cycle: finalizer tests repeatedly skipped GC and + exceeded their three-second deadlines. Runtime mutex/condition waits now + publish the suspended Go caller's roots before entering C and prevent a + return to Go until collection finishes. This includes the timer scheduler. + Blocked callers no longer wake every 20 ms: normal conditions wait for a + signal, and timers wait for their actual deadline. Arbitrary user C calls + retain the bounded-wait/skip-collection behavior. +- The initial thread parked after Goexit must unregister its roots, since it + will never execute another Go safepoint. A worker now verifies that GC still + advances after initial Goexit with the timer service active. +- Repeated explicit GC must allow waiting allocations and resumed threads to + progress. The allocator uses the same GC-safe pthread mutex instead of + spinning through repeated host calls, and the next collection waits for the + previous rendezvous to finish. Compiler polls and runtime symbol-table + initialization waiters acknowledge collection requests. Arbitrary user C + calls can still prevent collection. The 32-thread/1000-round concurrent + function-info test timed out at 100 seconds with the spin lock; the blocking + allocator completed it in 10.72 seconds (nogc comparison: 7.26 seconds) on + macOS arm64. These are local regression timings, not cross-host benchmarks. +- Small programs begin with a 1 MiB GC arena; subsequent arenas double up to + 32 MiB. This avoids sweeping a nearly empty 32 MiB arena on every explicit + collection. WAMR is built in Release mode, with the same classic interpreter, + exception and thread features used by the acceptance tests. +- Reflection's whole-program type-name lookup compares encoded names without + allocating a temporary string for each extra-star type. This preserves the + existing type identity and name rules while reducing GC contention. + +Safepoint checks use an atomic epoch read before acquiring the rendezvous +mutex. Heap scans carry the known segment through block-state operations; +address and block lookup use binary search, including out-of-order libc arenas. +The host metadata regression covers all 128 segment slots, gaps, sentinels, +metadata exclusion and preservation of marked objects during sweep. + +The focused `test/go` regression compiles once with a 300-second deadline. +It then runs three groups in fresh WAMR invocations: finalizers/callback GC/ +function-info, the 20 pointer-argument startup races, and the 20 zero-argument +startup races. Each invocation has a 300-second deadline and verbose output. +On Linux CI the startup groups took 134 and 145 seconds in one combined run, +while another run took 160 seconds for the pointer group and exceeded the +combined deadline in the zero-argument group. Separate deadlines retain every +case and repetition, distinguish slow interpreter work from a stalled test, +and keep first-use checks independent of earlier test initialization. + +## Reproducible checks + +```sh +bash dev/build_iwasm.sh +go test ./internal/build -run '^TestWASIGCWaitsSuspendGoAcrossMutexReacquisition$' +LLGO="$PWD/.bin/llgo" IWASM=/path/to/patched/iwasm python3 dev/test_wasm_wasi_threads.py +LLGO_WASI_THREADS=1 go run ./dev/wasmstdlib -full -profile W32-WASI \ + -shard 9 -shards 16 -llgo "$PWD/.bin/llgo" -report w32-shard9.json +``` + +The pthread regression exercises both mutex waits and condition-variable +reacquisition, including a waiter woken while collection is still active. +The WAMR acceptance driver covers GC/nogc exception isolation, main/init/worker +Goexit, finalizers, reflection with concurrent GC, retained roots, heap growth, +uncooperative C, timers, filesystems, selected standard-library tests and a +GOROOT sentinel. The full package audit remains a separate gate; passing this +focused suite is not a claim of complete standard-library compatibility. diff --git a/internal/build/build.go b/internal/build/build.go index 9763d4bfa8..0fc2a79119 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -680,8 +680,7 @@ func buildInvocation(inv Invocation, plan *initialBuildPlan) (result []Package, prog.EnableDeadcodeDrop(conf.deadcodeDropEnabled()) prog.EnableGCRoots(wasmGC) prog.EnableLogicalGoroutineLocality(usesSingleWorkerWasmScheduler(conf)) - prog.EnableThreadLocalGCRoots(wasmGC && - (wasmWorkers.Enabled() || conf.Goos == "wasip1" && IsWasiThreadsEnabled())) + prog.EnableThreadLocalGCRoots(useThreadLocalGCRoots(conf, wasmGC, wasmWorkers)) prog.EnableCooperativeSafepoints(wasmGC || wasmWorkers.Enabled()) if conf.PthreadStackSize > 0 { prog.SetPthreadStackSize(uint64(conf.PthreadStackSize)) @@ -1071,6 +1070,12 @@ func resolveTestPthreadStackSize(conf *Config) { } } +// Shared-memory collectors need one root chain per host thread. The +// single-worker scheduler and nogc builds keep their existing root model. +func useThreadLocalGCRoots(conf *Config, wasmGC bool, workers wasmworkers.Config) bool { + return wasmGC && (workers.Enabled() || conf.Goos == "wasip1" && IsWasiThreadsEnabled()) +} + func useShadowStack(goarch string) bool { // WebAssembly has no inspectable native return addresses. Its runtime // callers implementation therefore relies on LLGo's selective software diff --git a/internal/build/testdata/wasm-wasi-gc-wait/main.c b/internal/build/testdata/wasm-wasi-gc-wait/main.c new file mode 100644 index 0000000000..c31a98b32b --- /dev/null +++ b/internal/build/testdata/wasm-wasi-gc-wait/main.c @@ -0,0 +1,81 @@ +#define _POSIX_C_SOURCE 200809L +#include +#include +#include +#include +#include + +void llgo_wasi_gc_enter_begin(void); +void llgo_wasi_gc_enter_end(void); +void llgo_wasi_gc_leave_end(void); +int llgo_wasi_gc_stop(void); +void llgo_wasi_gc_resume(void); +void llgo_wasi_gc_mutex_lock(pthread_mutex_t *, uintptr_t, uintptr_t, uintptr_t); +void llgo_wasi_gc_cond_timedwait(pthread_cond_t *, pthread_mutex_t *, int64_t, + int, uintptr_t, uintptr_t, uintptr_t); + +static pthread_mutex_t application = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t condition = PTHREAD_COND_INITIALIZER; +static atomic_int published; +static atomic_int returned; + +void llgo_gcroot_publish_thread(uintptr_t chain, uintptr_t bottom, + uintptr_t top) { + assert(chain == 11 && bottom == 22 && top == 33); + atomic_store(&published, 1); +} +void llgo_gcroot_reset_thread_chain(void) {} +void *llgo_wasi_gc_mstart(void *arg) { return arg; } + +static void pause_briefly(void) { + struct timespec pause = {0, 1000000}; + nanosleep(&pause, 0); +} + +static void *waiter(void *arg) { + llgo_wasi_gc_enter_begin(); + llgo_wasi_gc_enter_end(); + if (arg) { + assert(pthread_mutex_lock(&application) == 0); + llgo_wasi_gc_cond_timedwait(&condition, &application, -1, + 0, 11, 22, 33); + } else { + llgo_wasi_gc_mutex_lock(&application, 11, 22, 33); + } + atomic_store(&returned, 1); + assert(pthread_mutex_unlock(&application) == 0); + llgo_wasi_gc_leave_end(); + return 0; +} + +int main(void) { + llgo_wasi_gc_enter_begin(); + llgo_wasi_gc_enter_end(); + for (int mode = 0; mode != 2; ++mode) { + for (int iteration = 0; iteration != 20; ++iteration) { + atomic_store(&published, 0); + atomic_store(&returned, 0); + if (!mode) + assert(pthread_mutex_lock(&application) == 0); + pthread_t thread; + assert(pthread_create(&thread, 0, waiter, (void *)(uintptr_t)mode) == 0); + while (!atomic_load(&published)) + pause_briefly(); + if (mode) + assert(pthread_mutex_lock(&application) == 0); + + // A stopped owner may hold the mutex a C waiter needs to reacquire. + // Collection must succeed, and waking C must not resume Go early. + assert(llgo_wasi_gc_stop()); + assert(pthread_cond_signal(&condition) == 0); + assert(pthread_mutex_unlock(&application) == 0); + pause_briefly(); + assert(!atomic_load(&returned)); + llgo_wasi_gc_resume(); + assert(pthread_join(thread, 0) == 0); + assert(atomic_load(&returned)); + } + } + llgo_wasi_gc_leave_end(); + return 0; +} diff --git a/internal/build/testdata/wasm-wasi-goexit-defer/_wrap/longjmp.c b/internal/build/testdata/wasm-wasi-goexit-defer/_wrap/longjmp.c new file mode 100644 index 0000000000..b1b8e84ba4 --- /dev/null +++ b/internal/build/testdata/wasm-wasi-goexit-defer/_wrap/longjmp.c @@ -0,0 +1,13 @@ +#include + +__attribute__((noinline)) static void jump(jmp_buf target) { + longjmp(target, 7); +} + +int llgo_wasi_thread_longjmp(void) { + jmp_buf target; + int value = setjmp(target); + if (!value) + jump(target); + return value; +} diff --git a/internal/build/testdata/wasm-wasi-goexit-defer/main.go b/internal/build/testdata/wasm-wasi-goexit-defer/main.go new file mode 100644 index 0000000000..e2c8a0931e --- /dev/null +++ b/internal/build/testdata/wasm-wasi-goexit-defer/main.go @@ -0,0 +1,82 @@ +package main + +import ( + "os" + "runtime" + _ "unsafe" +) + +const LLGoFiles = "_wrap/longjmp.c" + +//go:linkname cLongjmp C.llgo_wasi_thread_longjmp +func cLongjmp() int32 + +func init() { + if len(os.Args) > 1 && os.Args[1] == "init" { + defer println("wasi init defer ok") + runtime.Goexit() + } +} + +//go:noinline +func raise(depth int) { + if depth == 0 { + panic(42) + } + defer func() {}() + raise(depth - 1) +} + +func catch() { + defer func() { + if recover() != 42 { + panic("wrong recovered value") + } + }() + raise(3) +} + +//go:noinline +func exit() { runtime.Goexit() } + +func main() { + if len(os.Args) > 1 { + switch os.Args[1] { + case "main": + defer println("wasi main defer ok") + runtime.Goexit() + case "uncaught": + go func() { panic("wasi uncaught sentinel") }() + select {} + } + } + const workers = 4 + done := make(chan int, workers) + for range workers { + go func() { + for range 20 { + catch() + if cLongjmp() != 7 { + panic("C longjmp did not return to its own thread") + } + } + defer func() { + // A branch after catching the longjmp must not observe a + // transient cluster-wide termination flag. + sum := 0 + for i := range 8 { + sum += i + } + done <- sum + }() + exit() + }() + } + for range workers { + if <-done != 28 { + panic("defer did not run") + } + } + runtime.GC() + println("wasi worker defer ok") +} diff --git a/internal/build/testdata/wasm-wasi-goexit-defer/uncaught.wat b/internal/build/testdata/wasm-wasi-goexit-defer/uncaught.wat new file mode 100644 index 0000000000..80edcd73fb --- /dev/null +++ b/internal/build/testdata/wasm-wasi-goexit-defer/uncaught.wat @@ -0,0 +1,5 @@ +(module + (memory (export "memory") 1) + (tag $exception (param i32)) + (func $raise (throw $exception (i32.const 42))) + (func (export "_start") (call $raise))) diff --git a/internal/build/testdata/wasm-wasi-main-goexit-timer/gc.go b/internal/build/testdata/wasm-wasi-main-goexit-timer/gc.go new file mode 100644 index 0000000000..aa07cfd6f5 --- /dev/null +++ b/internal/build/testdata/wasm-wasi-main-goexit-timer/gc.go @@ -0,0 +1,15 @@ +//go:build !nogc + +package main + +import "runtime" + +func verifyWorkerGC() { + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + runtime.GC() + runtime.ReadMemStats(&after) + if after.NumGC <= before.NumGC { + panic("initial Goexit thread prevented worker GC") + } +} diff --git a/internal/build/testdata/wasm-wasi-main-goexit-timer/main.go b/internal/build/testdata/wasm-wasi-main-goexit-timer/main.go index ede64335b5..d9f52e2451 100644 --- a/internal/build/testdata/wasm-wasi-main-goexit-timer/main.go +++ b/internal/build/testdata/wasm-wasi-main-goexit-timer/main.go @@ -8,6 +8,7 @@ import ( func init() { go func() { time.Sleep(20 * time.Millisecond) + verifyWorkerGC() println("wasi goexit timer worker done") }() runtime.Goexit() diff --git a/internal/build/testdata/wasm-wasi-main-goexit-timer/nogc.go b/internal/build/testdata/wasm-wasi-main-goexit-timer/nogc.go new file mode 100644 index 0000000000..b2617a101a --- /dev/null +++ b/internal/build/testdata/wasm-wasi-main-goexit-timer/nogc.go @@ -0,0 +1,5 @@ +//go:build nogc + +package main + +func verifyWorkerGC() {} diff --git a/internal/build/wasm_gc_head_cache_test.go b/internal/build/wasm_gc_head_cache_test.go index ada9884c1e..4eb6f5945e 100644 --- a/internal/build/wasm_gc_head_cache_test.go +++ b/internal/build/wasm_gc_head_cache_test.go @@ -23,7 +23,8 @@ func TestWasmGCHeadCacheWorkloads(t *testing.T) { dir := filepath.Join("..", "..", "runtime", "internal", "runtime", "tinygogc") functions := map[string]bool{ "gcFindHead": true, "gcFindHeadForMark": true, - "gcStateByteOf": true, "gcStateFromByte": true, "gcStateOf": true, + "gcStateByteOfIn": true, "gcStateFromByteIn": true, "gcStateOf": true, + "gcFindHeadIn": true, "gcStateOfIn": true, } var source bytes.Buffer source.WriteString(gcHeadCacheWorkloadSource) diff --git a/internal/build/wasm_gc_roots_test.go b/internal/build/wasm_gc_roots_test.go new file mode 100644 index 0000000000..db16d05142 --- /dev/null +++ b/internal/build/wasm_gc_roots_test.go @@ -0,0 +1,30 @@ +package build + +import ( + "github.com/xgo-dev/llgo/internal/wasmworkers" + "testing" +) + +func TestThreadLocalGCRootSelection(t *testing.T) { + for _, tc := range []struct { + name, goos, threads string + gc bool + workers int + want bool + }{ + {"WASI GC", "wasip1", "1", true, 1, true}, + {"WASI nogc", "wasip1", "1", false, 1, false}, + {"browser workers", "js", "", true, 2, true}, + {"browser nogc", "js", "", false, 2, false}, + {"browser single worker", "js", "", true, 1, false}, + {"native", "linux", "1", false, 1, false}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("LLGO_WASI_THREADS", tc.threads) + got := useThreadLocalGCRoots(&Config{Goos: tc.goos}, tc.gc, wasmworkers.Config{Count: tc.workers}) + if got != tc.want { + t.Fatalf("thread-local roots = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/internal/build/wasm_gc_segments_test.go b/internal/build/wasm_gc_segments_test.go new file mode 100644 index 0000000000..bcf4340bba --- /dev/null +++ b/internal/build/wasm_gc_segments_test.go @@ -0,0 +1,120 @@ +package build + +import ( + "bytes" + "go/ast" + "go/format" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "testing" +) + +// Exercise the collector's real metadata code over out-of-order arenas without +// letting the host test's own collector share that metadata. +func TestWasmGCSegmentBoundariesAndSweep(t *testing.T) { + fset := token.NewFileSet() + var source bytes.Buffer + source.WriteString(gcSegmentWorkloadSource) + functions := map[string]bool{"gcAddressOfIn": true, "gcStateByteOfIn": true, + "gcStateFromByteIn": true, "gcStateOfIn": true, "gcSetStateIn": true, + "gcMarkFreeIn": true, "gcUnmarkIn": true, "sweep": true} + for _, name := range []string{"segments.go", "gc_tinygo.go"} { + file, err := parser.ParseFile(fset, filepath.Join("..", "..", "runtime", "internal", "runtime", "tinygogc", name), nil, 0) + if err != nil { + t.Fatal(err) + } + for _, decl := range file.Decls { + if gen, ok := decl.(*ast.GenDecl); ok && gen.Tok == token.IMPORT { + continue + } + if name == "gc_tinygo.go" { + fn, ok := decl.(*ast.FuncDecl) + if !ok || !functions[fn.Name.Name] { + continue + } + delete(functions, fn.Name.Name) + } + if err := format.Node(&source, fset, decl); err != nil { + t.Fatal(err) + } + source.WriteByte('\n') + } + } + if len(functions) != 0 { + t.Fatalf("missing collector functions: %v", functions) + } + path := filepath.Join(t.TempDir(), "segments_test.go") + if err := os.WriteFile(path, source.Bytes(), 0644); err != nil { + t.Fatal(err) + } + if out, err := exec.Command("go", "test", "-count=1", "-timeout=20s", path).CombinedOutput(); err != nil { + t.Fatalf("segment metadata regression: %v\n%s", err, out) + } +} + +const gcSegmentWorkloadSource = `package segments +import ("runtime"; "testing"; "unsafe") +const ( + blockStateFree uint8 = iota + blockStateHead + blockStateTail + blockStateMark + blockStateMask = 3 + stateBits = 2 + blocksPerStateByte = 4 + wordsPerBlock = 4 + bytesPerBlock = wordsPerBlock * unsafe.Sizeof(uintptr(0)) +) +var endBlock uintptr +var gcFrees, gcFreedBlocks uint64 +var c = struct { + Str func(string) string + Memset func(unsafe.Pointer, int, uintptr) unsafe.Pointer +}{func(s string) string { return s }, func(p unsafe.Pointer, value int, n uintptr) unsafe.Pointer { + for i := uintptr(0); i < n; i++ { *(*byte)(unsafe.Add(p, i)) = byte(value) }; return p +}} +func gcPanic(s string) { panic(s) } +func TestAllSegments(t *testing.T) { + const stride = 4096 + data := make([]byte, maxHeapSegments*stride) + base := uintptr(unsafe.Pointer(&data[0])) + for i := 0; i < maxHeapSegments; i++ { + // Force non-address-order insertion and holes between malloc-like arenas. + start := base + uintptr((i*37)%maxHeapSegments)*stride + if !addHeapSegment(start, start+stride-64) { t.Fatalf("segment %d rejected", i) } + } + for i := 0; i < heapSegmentCount; i++ { + segment := &heapSegments[i] + for block := segment.first; block <= segment.last; block++ { + if segmentForBlock(block) != segment { t.Fatalf("block %d crosses segment", block) } + } + for _, address := range []uintptr{segment.start, segment.metadata-1} { + if segmentForAddress(address) != segment { t.Fatalf("address %#x not in segment %d", address, i) } + } + for _, address := range []uintptr{segment.start-1, segment.metadata, segment.end-1, segment.end} { + if segmentForAddress(address) != nil { t.Fatalf("gap/metadata %#x treated as a root", address) } + } + for block := segment.first; block < segment.last; block++ { + gcSetStateIn(segment, block, blockStateHead) + *(*uintptr)(unsafe.Pointer(gcAddressOfIn(segment, block))) = 123 + } + gcSetStateIn(segment, segment.first, blockStateMark) + } + sweep() + for i := 0; i < heapSegmentCount; i++ { + segment := &heapSegments[i] + for block := segment.first; block < segment.last; block++ { + want := uint8(blockStateFree) + if block == segment.first { want = blockStateHead } + if got := gcStateOfIn(segment, block); got != want { t.Fatalf("segment %d block %d: %d != %d", i, block, got, want) } + value := *(*uintptr)(unsafe.Pointer(gcAddressOfIn(segment, block))) + if block == segment.first && value != 123 || block != segment.first && value != 0 { t.Fatalf("incorrect sweep payload at %d", block) } + } + } + if addHeapSegment(base, base+stride) { t.Fatal("segment limit ignored") } + runtime.KeepAlive(data) +} +` diff --git a/internal/build/wasm_wasi_gc_wait_test.go b/internal/build/wasm_wasi_gc_wait_test.go new file mode 100644 index 0000000000..4de997f1ca --- /dev/null +++ b/internal/build/wasm_wasi_gc_wait_test.go @@ -0,0 +1,30 @@ +//go:build darwin || linux + +package build + +import ( + stdcontext "context" + "os/exec" + "path/filepath" + "testing" + "time" +) + +func TestWASIGCWaitsSuspendGoAcrossMutexReacquisition(t *testing.T) { + clang, err := exec.LookPath("clang") + if err != nil { + t.Fatal(err) + } + bin := filepath.Join(t.TempDir(), "wasi-gc-wait") + args := []string{"-std=c11", "-O2", "-Wall", "-Wextra", "-Werror", "-pthread", + "testdata/wasm-wasi-gc-wait/main.c", + "../../runtime/internal/runtime/_wrap/wasi_gc_world.c", "-o", bin} + if out, err := exec.Command(clang, args...).CombinedOutput(); err != nil { + t.Fatalf("compile wait regression: %v\n%s", err, out) + } + ctx, cancel := stdcontext.WithTimeout(stdcontext.Background(), 10*time.Second) + defer cancel() + if out, err := exec.CommandContext(ctx, bin).CombinedOutput(); err != nil { + t.Fatalf("wait regression: %v\n%s", err, out) + } +} diff --git a/runtime/internal/lib/reflect/type.go b/runtime/internal/lib/reflect/type.go index 70a8f8a120..e5aba724d8 100644 --- a/runtime/internal/lib/reflect/type.go +++ b/runtime/internal/lib/reflect/type.go @@ -2604,9 +2604,17 @@ var typelist []*abi.Type func typesByString(s string) (typs []*abi.Type) { for _, t := range typelist { - if t.String() == s { - typs = append(typs, t) + // Type.String constructs a new string for ExtraStar types. Avoid an + // allocation for every pointer type in this whole-program lookup: a + // concurrent forced GC can otherwise stop the scan at each allocation. + if t.TFlag&abi.TFlagExtraStar != 0 { + if len(s) == 0 || s[0] != '*' || t.Str_ != s[1:] { + continue + } + } else if t.Str_ != s { + continue } + typs = append(typs, t) } return } diff --git a/runtime/internal/lib/runtime/time_gc_poll_default.go b/runtime/internal/lib/runtime/funcinfo_wait_default.go similarity index 61% rename from runtime/internal/lib/runtime/time_gc_poll_default.go rename to runtime/internal/lib/runtime/funcinfo_wait_default.go index d02592ec73..32a0564b25 100644 --- a/runtime/internal/lib/runtime/time_gc_poll_default.go +++ b/runtime/internal/lib/runtime/funcinfo_wait_default.go @@ -2,6 +2,4 @@ package runtime -const timerGCWaitQuantum = int64(0) - -func timerGCSafepoint() {} +func pollRuntimeTableWait() {} diff --git a/runtime/internal/lib/runtime/funcinfo_wait_wasi_gc.go b/runtime/internal/lib/runtime/funcinfo_wait_wasi_gc.go new file mode 100644 index 0000000000..74c5f1eb3d --- /dev/null +++ b/runtime/internal/lib/runtime/funcinfo_wait_wasi_gc.go @@ -0,0 +1,9 @@ +//go:build llgo && wasip1 && wasm && llgo.wasi_threads && llgo.wasm.gc.linear + +package runtime + +import llruntime "github.com/xgo-dev/llgo/runtime/internal/runtime" + +// The initializer may allocate while other threads wait for its publication. +// Those waiters must acknowledge GC instead of keeping it blocked in C sleep. +func pollRuntimeTableWait() { llruntime.CooperativeSafepoint() } diff --git a/runtime/internal/lib/runtime/symtab.go b/runtime/internal/lib/runtime/symtab.go index d9cfa8de85..bfae18619c 100644 --- a/runtime/internal/lib/runtime/symtab.go +++ b/runtime/internal/lib/runtime/symtab.go @@ -708,6 +708,7 @@ func initRuntimeFuncPCFramesSlow() { return } } + pollRuntimeTableWait() c.Usleep(1) } } @@ -853,6 +854,7 @@ func materializePrebuiltEntries() { latomic.StoreUint32(&runtimePrebuiltEntriesOnce, 2) return default: + pollRuntimeTableWait() c.Usleep(1) } } @@ -1590,6 +1592,7 @@ func initRuntimePCLineFramesSlow() { return } } + pollRuntimeTableWait() c.Usleep(1) } } diff --git a/runtime/internal/lib/runtime/time_gc_poll_wasi_llgo.go b/runtime/internal/lib/runtime/time_gc_poll_wasi_llgo.go deleted file mode 100644 index 75efeae7b1..0000000000 --- a/runtime/internal/lib/runtime/time_gc_poll_wasi_llgo.go +++ /dev/null @@ -1,9 +0,0 @@ -//go:build llgo && wasip1 && wasm && llgo.wasi_threads && llgo.wasm.gc.linear - -package runtime - -import llruntime "github.com/xgo-dev/llgo/runtime/internal/runtime" - -const timerGCWaitQuantum = int64(20 * 1e6) - -func timerGCSafepoint() { llruntime.CooperativeSafepoint() } diff --git a/runtime/internal/lib/runtime/time_heap_llgo.go b/runtime/internal/lib/runtime/time_heap_llgo.go index 88a38cbefa..7db22c8f2c 100644 --- a/runtime/internal/lib/runtime/time_heap_llgo.go +++ b/runtime/internal/lib/runtime/time_heap_llgo.go @@ -135,29 +135,14 @@ func timerSchedulerLoop() { markTimerSystemG() timerSchedulerMu.Lock() for { - // The WASI threaded collector needs the timer pthread to reach a Go - // safepoint even when no timers are due. Poll with the lock released so - // another thread waiting for this mutex can stop for collection too. - if timerGCWaitQuantum != 0 { - timerSchedulerMu.Unlock() - timerGCSafepoint() - timerSchedulerMu.Lock() - } if len(timerSchedulerHeap) == 0 { - if timerGCWaitQuantum == 0 { - timerSchedulerCond.Wait(&timerSchedulerMu) - } else { - timerSchedulerTimedWait(timerGCWaitQuantum) - } + timerSchedulerCond.Wait(&timerSchedulerMu) continue } st := timerSchedulerHeap[0] now := runtimeNano() if wait := timerSchedulerWaitDuration(st.r.when, now); wait > 0 { - if timerGCWaitQuantum != 0 && wait > timerGCWaitQuantum { - wait = timerGCWaitQuantum - } timerSchedulerTimedWait(wait) continue } diff --git a/runtime/internal/lib/runtime/time_wait_unix_llgo.go b/runtime/internal/lib/runtime/time_wait_unix_llgo.go index 53b9301da3..6859e3e3c4 100644 --- a/runtime/internal/lib/runtime/time_wait_unix_llgo.go +++ b/runtime/internal/lib/runtime/time_wait_unix_llgo.go @@ -1,4 +1,4 @@ -//go:build !baremetal && !windows && (!wasm || (wasip1 && llgo.wasi_threads)) +//go:build !baremetal && !windows && (!wasm || (wasip1 && llgo.wasi_threads && !llgo.wasm.gc.linear)) package runtime diff --git a/runtime/internal/lib/runtime/time_wait_wasi_gc_llgo.go b/runtime/internal/lib/runtime/time_wait_wasi_gc_llgo.go new file mode 100644 index 0000000000..4a59db3768 --- /dev/null +++ b/runtime/internal/lib/runtime/time_wait_wasi_gc_llgo.go @@ -0,0 +1,25 @@ +//go:build llgo && wasip1 && wasm && llgo.wasi_threads && llgo.wasm.gc.linear + +package runtime + +import ( + "unsafe" + + c "github.com/xgo-dev/llgo/runtime/internal/clite" + llruntime "github.com/xgo-dev/llgo/runtime/internal/runtime" + psync "github.com/xgo-dev/llgo/runtime/internal/sync" +) + +//go:linkname c_timerCondInit C.llgo_timer_cond_init +func c_timerCondInit(cond *psync.Cond) c.Int + +func initTimerSchedulerCond() { + if c_timerCondInit(&timerSchedulerCond) != 0 { + panic("runtime: failed to initialize timer condition variable") + } +} + +func timerSchedulerTimedWait(wait int64) { + llruntime.WasiGCCondTimedWait(unsafe.Pointer(&timerSchedulerCond), + unsafe.Pointer(&timerSchedulerMu), wait, true) +} diff --git a/runtime/internal/runtime/_wrap/wasi_gc_world.c b/runtime/internal/runtime/_wrap/wasi_gc_world.c index 5a45c032b8..33f9b1f406 100644 --- a/runtime/internal/runtime/_wrap/wasi_gc_world.c +++ b/runtime/internal/runtime/_wrap/wasi_gc_world.c @@ -2,6 +2,7 @@ #include #include #include +#include #include // Count each entering thread before it registers Go roots and keep it counted @@ -9,12 +10,18 @@ // holding this mutex across Go allocation or TLS setup. static pthread_mutex_t world_mutex = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t world_changed = PTHREAD_COND_INITIALIZER; -static uint32_t world_epoch; +static _Atomic uint32_t world_epoch; +// A C call that cannot rendezvous aborts this collection instead of sweeping +// a running stack. This is a deadline, not a periodic mutator wakeup. +#define LLGO_WASI_GC_STOP_TIMEOUT_NS 500000000L static uint32_t world_registered; static uint32_t world_stopped; +static uint32_t world_blocked; +static uint32_t world_resuming; static pthread_t world_owner; static _Thread_local uint32_t world_seen_epoch; static _Thread_local int world_is_registered; +static _Thread_local int world_is_blocked; extern void llgo_gcroot_publish_thread(uintptr_t chain, uintptr_t bottom, uintptr_t top); @@ -41,14 +48,65 @@ static void world_wait(void) { __builtin_trap(); } -// Wake hosted Go condition waiters often enough to publish their roots for a -// stop-the-world request, even when no application signal is forthcoming. +// These waits only access pthread state. Publish the suspended Go caller's +// roots before entering C and prevent its return to Go during collection. +// In particular, cond_wait may block while reacquiring a mutex whose owner +// has already stopped at an allocation safepoint. +static void world_begin_wait(uintptr_t chain, uintptr_t bottom, uintptr_t top) { + if (!world_is_registered) + return; + llgo_gcroot_publish_thread(chain, bottom, top); + world_lock(); + if (world_is_blocked) + __builtin_trap(); + world_is_blocked = 1; + world_blocked++; + if (pthread_cond_broadcast(&world_changed) != 0) + __builtin_trap(); + world_unlock(); +} + +static void world_end_wait(void) { + if (!world_is_registered) + return; + world_lock(); + world_resuming++; + while (world_epoch & 1) + world_wait(); + if (!world_is_blocked || world_blocked == 0) + __builtin_trap(); + world_blocked--; + world_is_blocked = 0; + world_resuming--; + if (pthread_cond_broadcast(&world_changed) != 0) + __builtin_trap(); + world_unlock(); +} + +void llgo_wasi_gc_mutex_lock(pthread_mutex_t *mutex, uintptr_t chain, + uintptr_t bottom, uintptr_t top) { + world_begin_wait(chain, bottom, top); + if (pthread_mutex_lock(mutex) != 0) + __builtin_trap(); + world_end_wait(); +} + void llgo_wasi_gc_cond_timedwait(pthread_cond_t *condition, - pthread_mutex_t *mutex) { + pthread_mutex_t *mutex, int64_t wait_nanos, + int monotonic, uintptr_t chain, + uintptr_t bottom, uintptr_t top) { + world_begin_wait(chain, bottom, top); + if (wait_nanos < 0) { + if (pthread_cond_wait(condition, mutex) != 0) + __builtin_trap(); + world_end_wait(); + return; + } struct timespec deadline; - if (clock_gettime(CLOCK_REALTIME, &deadline) != 0) + if (clock_gettime(monotonic ? CLOCK_MONOTONIC : CLOCK_REALTIME, &deadline) != 0) __builtin_trap(); - deadline.tv_nsec += 20000000; + deadline.tv_sec += (time_t)(wait_nanos / 1000000000); + deadline.tv_nsec += (long)(wait_nanos % 1000000000); if (deadline.tv_nsec >= 1000000000) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000; @@ -56,6 +114,7 @@ void llgo_wasi_gc_cond_timedwait(pthread_cond_t *condition, int status = pthread_cond_timedwait(condition, mutex, &deadline); if (status != 0 && status != ETIMEDOUT) __builtin_trap(); + world_end_wait(); } void llgo_wasi_gc_enter_begin(void) { @@ -91,6 +150,10 @@ void llgo_wasi_gc_leave_end(void) { } int llgo_wasi_gc_pending(void) { + // The overwhelmingly common case needs no process-wide pthread lock. + // Only inspect world_owner while holding the lock after observing a stop. + if (!(atomic_load_explicit(&world_epoch, memory_order_acquire) & 1)) + return 0; world_lock(); int pending = (world_epoch & 1) && !pthread_equal(world_owner, pthread_self()); @@ -108,6 +171,7 @@ int llgo_wasi_gc_registered(void) { void llgo_wasi_gc_park(uintptr_t chain, uintptr_t bottom, uintptr_t top) { llgo_gcroot_publish_thread(chain, bottom, top); world_lock(); + world_resuming++; while ((world_epoch & 1) && !pthread_equal(world_owner, pthread_self())) { uint32_t epoch = world_epoch; if (world_seen_epoch != epoch) { @@ -119,6 +183,9 @@ void llgo_wasi_gc_park(uintptr_t chain, uintptr_t bottom, uintptr_t top) { while (world_epoch == epoch) world_wait(); } + world_resuming--; + if (pthread_cond_broadcast(&world_changed) != 0) + __builtin_trap(); world_unlock(); } @@ -129,6 +196,10 @@ int llgo_wasi_gc_stop(void) { world_lock(); if ((world_epoch & 1) || !world_is_registered || world_registered == 0) __builtin_trap(); + // A new collection must let waiters leave the previous rendezvous first. + // Otherwise repeated runtime.GC calls can re-park them without Go progress. + while (world_resuming != 0) + world_wait(); world_owner = pthread_self(); world_stopped = 0; world_epoch++; @@ -136,12 +207,13 @@ int llgo_wasi_gc_stop(void) { struct timespec deadline; if (clock_gettime(CLOCK_REALTIME, &deadline) != 0) __builtin_trap(); - deadline.tv_nsec += 500000000; + deadline.tv_nsec += LLGO_WASI_GC_STOP_TIMEOUT_NS; if (deadline.tv_nsec >= 1000000000) { deadline.tv_sec++; deadline.tv_nsec -= 1000000000; } - while (world_stopped < world_registered - (uint32_t)world_is_registered) { + while (world_stopped + world_blocked < + world_registered - (uint32_t)world_is_registered) { int status = pthread_cond_timedwait(&world_changed, &world_mutex, &deadline); if (status == ETIMEDOUT) { diff --git a/runtime/internal/runtime/goexit_initial_wasi_threads.go b/runtime/internal/runtime/goexit_initial_wasi_threads.go index adc4288044..4a4eceaf8d 100644 --- a/runtime/internal/runtime/goexit_initial_wasi_threads.go +++ b/runtime/internal/runtime/goexit_initial_wasi_threads.go @@ -11,10 +11,10 @@ func parkInitialWasiThread(gp *g) { releaseStartArg(gp) casgstatus(gp, _Grunning, _Gdead) releaseGAndCheckDeadlock() + // This thread will never return to Go. Leaving it registered would make + // every later worker collection time out waiting for its safepoint. + unregisterWasiGCThread() for { - // This pthread remains registered with the collector even though its - // main G is dead. Let a surviving worker stop it for collection. - CooperativeSafepoint() c.Usleep(1000) } } diff --git a/runtime/internal/runtime/tinygogc/_wrap/gc_wasm.c b/runtime/internal/runtime/tinygogc/_wrap/gc_wasm.c index dbf0664dfc..b0fe955e53 100644 --- a/runtime/internal/runtime/tinygogc/_wrap/gc_wasm.c +++ b/runtime/internal/runtime/tinygogc/_wrap/gc_wasm.c @@ -25,7 +25,9 @@ extern unsigned char __heap_base; // disjoint region before tinygogc starts, rather than treating all remaining // linear memory as a Go heap and corrupting later libc allocations. Further // disjoint regions can be allocated as the Go heap grows. -#define LLGO_WASI_GC_ARENA_SIZE (32u << 20) +#define LLGO_WASI_GC_ARENA_SIZE (1u << 20) +// These bounds describe only the first malloc-backed segment. Later arenas +// have independent descriptors in tinygogc; they do not extend this region. static uintptr_t llgo_wasi_gc_arena_start; static uintptr_t llgo_wasi_gc_arena_end; @@ -101,6 +103,7 @@ int llgo_gc_grow_memory(uintptr_t required) { #if defined(__EMSCRIPTEN__) return emscripten_resize_heap(required); #elif defined(__wasi__) && defined(_REENTRANT) + // WASI growth uses llgo_gc_new_arena, never in-place memory.grow. return required <= llgo_gc_memory_size(); #else uintptr_t current = llgo_gc_memory_size(); diff --git a/runtime/internal/runtime/tinygogc/gc_segment_wasi_threads.go b/runtime/internal/runtime/tinygogc/gc_segment_wasi_threads.go index c37f4952e4..ec66e8f3a4 100644 --- a/runtime/internal/runtime/tinygogc/gc_segment_wasi_threads.go +++ b/runtime/internal/runtime/tinygogc/gc_segment_wasi_threads.go @@ -7,8 +7,13 @@ import _ "unsafe" const segmentedHeap = true func newHeapSegment(minimum uintptr) (uintptr, uintptr) { - const segmentSize = uintptr(32 << 20) - size := segmentSize + // Begin with a small heap so explicit GC does not sweep 32 MiB for tiny + // programs. Double successive arenas up to the former 32 MiB size. + shift := heapSegmentCount + if shift > 5 { + shift = 5 + } + size := uintptr(1<<20) << shift // The nominal arena size includes its metadata. Account for that and // alignment at both ends before comparing with the default arena size; // even an object slightly smaller than 32 MiB may not fit in that arena. diff --git a/runtime/internal/runtime/tinygogc/gc_tinygo.go b/runtime/internal/runtime/tinygogc/gc_tinygo.go index 0b8bbf24f0..2274ec8d59 100644 --- a/runtime/internal/runtime/tinygogc/gc_tinygo.go +++ b/runtime/internal/runtime/tinygogc/gc_tinygo.go @@ -109,14 +109,15 @@ func initGC() { metadataStart = unsafe.Pointer(heapSegments[0].metadata) } +// Contiguous (browser/bare-metal) growth moves existing metadata; unlike +// addHeapSegment it must not zero the old allocation state. func configureHeap() { - totalSize := heapEnd - heapStart - metadataSize := (totalSize + blocksPerStateByte*bytesPerBlock) / (1 + blocksPerStateByte*bytesPerBlock) - metadataStart = unsafe.Pointer(heapEnd - metadataSize) - endBlock = (uintptr(metadataStart) - heapStart) / bytesPerBlock - heapSegments[0].end = heapEnd - heapSegments[0].metadata = uintptr(metadataStart) - heapSegments[0].last = endBlock + segment := &heapSegments[0] + if !segment.configure(heapEnd) { + gcPanic(c.Str("gc: invalid grown heap segment")) + } + metadataStart = unsafe.Pointer(segment.metadata) + endBlock = segment.last } func lazyInit() { @@ -148,7 +149,10 @@ func gcPointerOf(blockAddr uintptr) unsafe.Pointer { // Return the address of the start of the allocated object. func gcAddressOf(blockAddr uintptr) uintptr { - segment := segmentForBlock(blockAddr) + return gcAddressOfIn(segmentForBlock(blockAddr), blockAddr) +} + +func gcAddressOfIn(segment *heapSegment, blockAddr uintptr) uintptr { addr := segment.start + (blockAddr-segment.first)*bytesPerBlock if addr > segment.metadata { gcPanic(c.Str("gc: block pointing inside metadata")) @@ -160,7 +164,10 @@ func gcAddressOf(blockAddr uintptr) uintptr { // points to an allocated object. It returns the same block if this block // already points to the head. func gcFindHead(blockAddr uintptr) uintptr { - segment := segmentForBlock(blockAddr) + return gcFindHeadIn(segmentForBlock(blockAddr), blockAddr) +} + +func gcFindHeadIn(segment *heapSegment, blockAddr uintptr) uintptr { for { // Optimization: check whether the current block state byte (which // contains the state of multiple blocks) is composed entirely of tail @@ -168,7 +175,7 @@ func gcFindHead(blockAddr uintptr) uintptr { // state byte. // This optimization speeds up findHead for pointers that point into a // large allocation. - stateByte := gcStateByteOf(blockAddr) + stateByte := gcStateByteOfIn(segment, blockAddr) if stateByte == blockStateByteAllTails { blockAddr -= ((blockAddr - segment.first) % blocksPerStateByte) + 1 continue @@ -176,13 +183,13 @@ func gcFindHead(blockAddr uintptr) uintptr { // Check whether we've found a non-tail block, which means we found the // head. - state := gcStateFromByte(blockAddr, stateByte) + state := gcStateFromByteIn(segment, blockAddr, stateByte) if state != blockStateTail { break } blockAddr-- } - if gcStateOf(blockAddr) != blockStateHead && gcStateOf(blockAddr) != blockStateMark { + if gcStateOfIn(segment, blockAddr) != blockStateHead && gcStateOfIn(segment, blockAddr) != blockStateMark { gcPanic(c.Str("gc: found tail without head")) } return blockAddr @@ -206,67 +213,89 @@ func gcFindHeadForMark(block uintptr) uintptr { // findNext returns the first block just past the end of the tail. This may or // may not be the head of an object. func gcFindNext(blockAddr uintptr) uintptr { - segment := segmentForBlock(blockAddr) - if gcStateOf(blockAddr) == blockStateHead || gcStateOf(blockAddr) == blockStateMark { + return gcFindNextIn(segmentForBlock(blockAddr), blockAddr) +} + +func gcFindNextIn(segment *heapSegment, blockAddr uintptr) uintptr { + if gcStateOfIn(segment, blockAddr) == blockStateHead || gcStateOfIn(segment, blockAddr) == blockStateMark { blockAddr++ } - for blockAddr < segment.last && gcStateOf(blockAddr) == blockStateTail { + for blockAddr < segment.last && gcStateOfIn(segment, blockAddr) == blockStateTail { blockAddr++ } return blockAddr } func gcStateByteOf(blockAddr uintptr) byte { - segment := segmentForBlock(blockAddr) + return gcStateByteOfIn(segmentForBlock(blockAddr), blockAddr) +} + +func gcStateByteOfIn(segment *heapSegment, blockAddr uintptr) byte { return *(*uint8)(unsafe.Pointer(segment.metadata + (blockAddr-segment.first)/blocksPerStateByte)) } // Return the block state given a state byte. The state byte must have been // obtained using b.stateByte(), otherwise the result is incorrect. func gcStateFromByte(blockAddr uintptr, stateByte byte) uint8 { - segment := segmentForBlock(blockAddr) + return gcStateFromByteIn(segmentForBlock(blockAddr), blockAddr, stateByte) +} + +func gcStateFromByteIn(segment *heapSegment, blockAddr uintptr, stateByte byte) uint8 { return uint8(stateByte>>(((blockAddr-segment.first)%blocksPerStateByte)*stateBits)) & blockStateMask } // State returns the current block state. func gcStateOf(blockAddr uintptr) uint8 { - return gcStateFromByte(blockAddr, gcStateByteOf(blockAddr)) + return gcStateOfIn(segmentForBlock(blockAddr), blockAddr) +} + +func gcStateOfIn(segment *heapSegment, blockAddr uintptr) uint8 { + return gcStateFromByteIn(segment, blockAddr, gcStateByteOfIn(segment, blockAddr)) } // setState sets the current block to the given state, which must contain more // bits than the current state. Allowed transitions: from free to any state and // from head to mark. func gcSetState(blockAddr uintptr, newState uint8) { - segment := segmentForBlock(blockAddr) + gcSetStateIn(segmentForBlock(blockAddr), blockAddr, newState) +} + +func gcSetStateIn(segment *heapSegment, blockAddr uintptr, newState uint8) { stateBytePtr := (*uint8)(unsafe.Pointer(segment.metadata + (blockAddr-segment.first)/blocksPerStateByte)) *stateBytePtr |= uint8(newState << (((blockAddr - segment.first) % blocksPerStateByte) * stateBits)) - if gcStateOf(blockAddr) != newState { + if gcStateOfIn(segment, blockAddr) != newState { gcPanic(c.Str("gc: setState() was not successful")) } } // markFree sets the block state to free, no matter what state it was in before. func gcMarkFree(blockAddr uintptr) { - segment := segmentForBlock(blockAddr) + gcMarkFreeIn(segmentForBlock(blockAddr), blockAddr) +} + +func gcMarkFreeIn(segment *heapSegment, blockAddr uintptr) { stateBytePtr := (*uint8)(unsafe.Pointer(segment.metadata + (blockAddr-segment.first)/blocksPerStateByte)) *stateBytePtr &^= uint8(blockStateMask << (((blockAddr - segment.first) % blocksPerStateByte) * stateBits)) - if gcStateOf(blockAddr) != blockStateFree { + if gcStateOfIn(segment, blockAddr) != blockStateFree { gcPanic(c.Str("gc: markFree() was not successful")) } - *(*[wordsPerBlock]uintptr)(unsafe.Pointer(gcAddressOf(blockAddr))) = [wordsPerBlock]uintptr{} + *(*[wordsPerBlock]uintptr)(unsafe.Pointer(gcAddressOfIn(segment, blockAddr))) = [wordsPerBlock]uintptr{} } // unmark changes the state of the block from mark to head. It must be marked // before calling this function. func gcUnmark(blockAddr uintptr) { - if gcStateOf(blockAddr) != blockStateMark { + gcUnmarkIn(segmentForBlock(blockAddr), blockAddr) +} + +func gcUnmarkIn(segment *heapSegment, blockAddr uintptr) { + if gcStateOfIn(segment, blockAddr) != blockStateMark { gcPanic(c.Str("gc: unmark() on a block that is not marked")) } clearMask := blockStateMask ^ blockStateHead // the bits to clear from the state - segment := segmentForBlock(blockAddr) stateBytePtr := (*uint8)(unsafe.Pointer(segment.metadata + (blockAddr-segment.first)/blocksPerStateByte)) *stateBytePtr &^= uint8(clearMask << (((blockAddr - segment.first) % blocksPerStateByte) * stateBits)) - if gcStateOf(blockAddr) != blockStateHead { + if gcStateOfIn(segment, blockAddr) != blockStateHead { gcPanic(c.Str("gc: unmark() was not successful")) } } @@ -300,6 +329,7 @@ func Alloc(size uintptr) unsafe.Pointer { // Continue looping until a run of free blocks has been found that fits the // requested size. index := nextAlloc + segment := segmentForBlock(index) numFreeBlocks := uintptr(0) heapScanCount := uint8(0) for { @@ -338,8 +368,13 @@ func Alloc(size uintptr) unsafe.Pointer { } // Wrap around the end of the heap. - if next, atEnd := nextSegmentBlock(index); atEnd { - index = next + if index == segment.last { + if index == endBlock { + index = 0 + } else { + index++ + } + segment = segmentForBlock(index) // Reset numFreeBlocks as allocations cannot wrap. numFreeBlocks = 0 // In rare cases, the initial heap might be so small that there are @@ -352,7 +387,7 @@ func Alloc(size uintptr) unsafe.Pointer { } // Is the block we're looking at free? - if gcStateOf(index) != blockStateFree { + if gcStateOfIn(segment, index) != blockStateFree { // This block is in use. Try again from this point. numFreeBlocks = 0 index++ @@ -368,11 +403,11 @@ func Alloc(size uintptr) unsafe.Pointer { thisAlloc := index - neededBlocks // Set the following blocks as being allocated. - gcSetState(thisAlloc, blockStateHead) + gcSetStateIn(segment, thisAlloc, blockStateHead) for i := thisAlloc + 1; i != nextAlloc; i++ { - gcSetState(i, blockStateTail) + gcSetStateIn(segment, i, blockStateTail) } - ret := c.Memset(gcPointerOf(thisAlloc), 0, size) + ret := c.Memset(unsafe.Pointer(gcAddressOfIn(segment, thisAlloc)), 0, size) unlock(&gcMutex) scheduleFinalizers() // Return a pointer to this allocation. @@ -514,23 +549,25 @@ func startMark(root uintptr) { stackLen-- block := stack[stackLen] - endBlock := gcFindNext(block) + segment := segmentForBlock(block) + endBlock := gcFindNextIn(segment, block) markHeads.remember(block, endBlock) - start, end := gcAddressOf(block), gcAddressOf(endBlock) + start, end := gcAddressOfIn(segment, block), gcAddressOfIn(segment, endBlock) for addr := start; addr != end; addr += gcScanWordSize { // Load the word. word := loadGCScanWord(addr) - if !isPointer(word) { + referencedSegment := segmentForAddress(word) + if referencedSegment == nil { // Not a heap pointer. continue } // Find the corresponding memory block. - referencedBlock := blockFromAddr(word) + referencedBlock := referencedSegment.first + (word-referencedSegment.start)/bytesPerBlock - if gcStateOf(referencedBlock) == blockStateFree { + if gcStateOfIn(referencedSegment, referencedBlock) == blockStateFree { // The to-be-marked object doesn't actually exist. // This is probably a false positive. continue @@ -540,13 +577,13 @@ func startMark(root uintptr) { referencedBlock = gcFindHeadForMark(referencedBlock) noteFinalizerReference(referencedBlock) - if gcStateOf(referencedBlock) == blockStateMark { + if gcStateOfIn(referencedSegment, referencedBlock) == blockStateMark { // The block has already been marked by something else. continue } // Mark block. - gcSetState(referencedBlock, blockStateMark) + gcSetStateIn(referencedSegment, referencedBlock, blockStateMark) if stackLen == len(stack) { // The stack is full. @@ -573,7 +610,7 @@ func finishMark() { for segmentIndex := 0; segmentIndex < heapSegmentCount; segmentIndex++ { segment := &heapSegments[segmentIndex] for block := segment.first; block < segment.last; block++ { - if gcStateOf(block) != blockStateMark { + if gcStateOfIn(segment, block) != blockStateMark { continue } startMark(block) @@ -609,10 +646,10 @@ func sweep() (freeBytes uintptr) { for segmentIndex := 0; segmentIndex < heapSegmentCount; segmentIndex++ { segment := &heapSegments[segmentIndex] for block := segment.first; block < segment.last; block++ { - switch gcStateOf(block) { + switch gcStateOfIn(segment, block) { case blockStateHead: // Unmarked head. Free it, including all tail blocks following it. - gcMarkFree(block) + gcMarkFreeIn(segment, block) freeCurrentObject = true gcFrees++ freed++ @@ -620,14 +657,14 @@ func sweep() (freeBytes uintptr) { if freeCurrentObject { // This is a tail object following an unmarked head. // Free it now. - gcMarkFree(block) + gcMarkFreeIn(segment, block) freed++ } case blockStateMark: // This is a marked object. The next tail blocks must not be freed, // but the mark bit must be removed so the next GC cycle will // collect this object if it is unreferenced then. - gcUnmark(block) + gcUnmarkIn(segment, block) freeCurrentObject = false case blockStateFree: freeBytes += bytesPerBlock diff --git a/runtime/internal/runtime/tinygogc/mutex_wasi_threads.go b/runtime/internal/runtime/tinygogc/mutex_wasi_threads.go index 86a2f19fed..9267b13f16 100644 --- a/runtime/internal/runtime/tinygogc/mutex_wasi_threads.go +++ b/runtime/internal/runtime/tinygogc/mutex_wasi_threads.go @@ -2,29 +2,15 @@ package tinygogc -import ( - _ "unsafe" +import "github.com/xgo-dev/llgo/runtime/internal/sync" - c "github.com/xgo-dev/llgo/runtime/internal/clite" - "github.com/xgo-dev/llgo/runtime/internal/sync/atomic" -) +// wasi-libc's PTHREAD_MUTEX_INITIALIZER is all zero, so the global allocator +// lock is usable before Go package initialization. Runtime Mutex.Lock publishes +// roots while waiting in C, letting the collector stop allocation waiters. +// Sleeping on the host futex avoids a sched_yield/host-mutex polling storm +// when many pthreads allocate concurrently in WAMR's interpreter. +type mutex = sync.Mutex -type mutex struct{ state uint32 } +func lock(m *mutex) { m.Lock() } -func lock(m *mutex) { - for { - if _, ok := atomic.CompareAndExchange(&m.state, uint32(0), uint32(1)); ok { - return - } - wasiGCAllocatorYield() - _ = wasiThreadYield() - } -} - -func unlock(m *mutex) { atomic.Store(&m.state, uint32(0)) } - -//go:linkname wasiGCAllocatorYield github.com/xgo-dev/llgo/runtime/internal/runtime.wasiGCSafepoint -func wasiGCAllocatorYield() - -//go:linkname wasiThreadYield C.sched_yield -func wasiThreadYield() c.Int +func unlock(m *mutex) { m.Unlock() } diff --git a/runtime/internal/runtime/tinygogc/segments.go b/runtime/internal/runtime/tinygogc/segments.go index 796a872724..764b9104a9 100644 --- a/runtime/internal/runtime/tinygogc/segments.go +++ b/runtime/internal/runtime/tinygogc/segments.go @@ -21,6 +21,19 @@ const maxHeapSegments = 128 var heapSegments [maxHeapSegments]heapSegment var heapSegmentCount int +// libc arenas need not be returned in address order. Keep a separate sorted +// index for conservative pointer lookup; logical block indices stay append-only. +var heapSegmentsByAddress [maxHeapSegments]*heapSegment + +func (segment *heapSegment) configure(end uintptr) bool { + segment.end = end + totalSize := end - segment.start + metadataSize := (totalSize + blocksPerStateByte*bytesPerBlock) / (1 + blocksPerStateByte*bytesPerBlock) + segment.metadata = end - metadataSize + segment.last = segment.first + (segment.metadata-segment.start)/bytesPerBlock + return segment.last > segment.first +} + func addHeapSegment(start, end uintptr) bool { if heapSegmentCount == maxHeapSegments || start >= end { return false @@ -33,13 +46,15 @@ func addHeapSegment(start, end uintptr) bool { if heapSegmentCount != 0 { segment.first = endBlock + 1 } - totalSize := end - start - metadataSize := (totalSize + blocksPerStateByte*bytesPerBlock) / (1 + blocksPerStateByte*bytesPerBlock) - segment.metadata = end - metadataSize - segment.last = segment.first + (segment.metadata-start)/bytesPerBlock - if segment.last <= segment.first || segment.last < segment.first { + if !segment.configure(end) { return false } + position := heapSegmentCount + for position > 0 && heapSegmentsByAddress[position-1].start > start { + heapSegmentsByAddress[position] = heapSegmentsByAddress[position-1] + position-- + } + heapSegmentsByAddress[position] = segment c.Memset(unsafe.Pointer(segment.metadata), 0, end-segment.metadata) heapSegmentCount++ endBlock = segment.last @@ -47,9 +62,15 @@ func addHeapSegment(start, end uintptr) bool { } func segmentForBlock(block uintptr) *heapSegment { - for index := 0; index < heapSegmentCount; index++ { - segment := &heapSegments[index] - if block >= segment.first && block <= segment.last { + low, high := 0, heapSegmentCount + for low < high { + mid := low + (high-low)/2 + segment := &heapSegments[mid] + if block < segment.first { + high = mid + } else if block > segment.last { + low = mid + 1 + } else { return segment } } @@ -58,25 +79,22 @@ func segmentForBlock(block uintptr) *heapSegment { } func segmentForAddress(address uintptr) *heapSegment { - for index := 0; index < heapSegmentCount; index++ { - segment := &heapSegments[index] - if address >= segment.start && address < segment.metadata { - return segment - } - } - return nil -} - -func nextSegmentBlock(block uintptr) (uintptr, bool) { - for index := 0; index < heapSegmentCount; index++ { - if heapSegments[index].last == block { - if index+1 < heapSegmentCount { - return heapSegments[index+1].first, true + low, high := 0, heapSegmentCount + for low < high { + mid := low + (high-low)/2 + segment := heapSegmentsByAddress[mid] + if address < segment.start { + high = mid + } else if address >= segment.end { + low = mid + 1 + } else { + if address < segment.metadata { + return segment } - return 0, true + return nil } } - return 0, false + return nil } func heapUsableSize() uintptr { diff --git a/runtime/internal/runtime/wasi_gc_threads.go b/runtime/internal/runtime/wasi_gc_threads.go index 6ce60fa2f9..893787dfde 100644 --- a/runtime/internal/runtime/wasi_gc_threads.go +++ b/runtime/internal/runtime/wasi_gc_threads.go @@ -48,6 +48,29 @@ func wasiGCSafepoint() { wasiGCPark(uintptr(gcroot.CurrentChain()), bottom, top) } +func wasiGCMutexLock(mutex unsafe.Pointer) { + wasiGCMutexWait(mutex, uintptr(gcroot.CurrentChain()), + uintptr(wasiGCStackPointer()), wasiGCStackTop()) +} + +// WasiGCCondTimedWait keeps the Go caller suspended while a pthread condition +// wait releases and reacquires its lock. Only the C pthread state may change. +// A negative waitNanos waits for a signal without a timer. +func WasiGCCondTimedWait(cond, mutex unsafe.Pointer, waitNanos int64, monotonic bool) { + var clock c.Int + if monotonic { + clock = 1 + } + wasiGCCondWait(cond, mutex, waitNanos, clock, uintptr(gcroot.CurrentChain()), + uintptr(wasiGCStackPointer()), wasiGCStackTop()) +} + +//go:linkname wasiGCMutexWait C.llgo_wasi_gc_mutex_lock +func wasiGCMutexWait(mutex unsafe.Pointer, chain, bottom, top uintptr) + +//go:linkname wasiGCCondWait C.llgo_wasi_gc_cond_timedwait +func wasiGCCondWait(cond, mutex unsafe.Pointer, waitNanos int64, monotonic c.Int, chain, bottom, top uintptr) + //go:linkname wasiGCEnterBegin C.llgo_wasi_gc_enter_begin func wasiGCEnterBegin() diff --git a/runtime/internal/sync/sync_unix.go b/runtime/internal/sync/sync_unix.go index f721390954..660d796eba 100644 --- a/runtime/internal/sync/sync_unix.go +++ b/runtime/internal/sync/sync_unix.go @@ -82,7 +82,7 @@ func (m *Mutex) Destroy() { } func (m *Mutex) Lock() { - pthreadMutexLock(m) + mutexLock(m) } func (m *Mutex) Unlock() { diff --git a/runtime/internal/sync/sync_wait_default.go b/runtime/internal/sync/sync_wait_default.go index fbdd17f860..0bad1b85ec 100644 --- a/runtime/internal/sync/sync_wait_default.go +++ b/runtime/internal/sync/sync_wait_default.go @@ -4,6 +4,8 @@ package sync import c "github.com/xgo-dev/llgo/runtime/internal/clite" +func mutexLock(m *Mutex) { pthreadMutexLock(m) } + func condWait(cond *Cond, mutex *Mutex) c.Int { return pthreadCondWait(cond, mutex) } diff --git a/runtime/internal/sync/sync_wait_wasi_threads_gc.go b/runtime/internal/sync/sync_wait_wasi_threads_gc.go index 6e73acf18b..5a1c6fe054 100644 --- a/runtime/internal/sync/sync_wait_wasi_threads_gc.go +++ b/runtime/internal/sync/sync_wait_wasi_threads_gc.go @@ -3,24 +3,20 @@ package sync import ( - _ "unsafe" + "unsafe" c "github.com/xgo-dev/llgo/runtime/internal/clite" ) -//go:linkname wasiGCCondTimedWait C.llgo_wasi_gc_cond_timedwait -func wasiGCCondTimedWait(cond *Cond, mutex *Mutex) +//go:linkname wasiGCCondTimedWait github.com/xgo-dev/llgo/runtime/internal/runtime.WasiGCCondTimedWait +func wasiGCCondTimedWait(cond, mutex unsafe.Pointer, waitNanos int64, monotonic bool) -//go:linkname wasiGCSafepoint github.com/xgo-dev/llgo/runtime/internal/runtime.wasiGCSafepoint -func wasiGCSafepoint() +//go:linkname mutexLock github.com/xgo-dev/llgo/runtime/internal/runtime.wasiGCMutexLock +func mutexLock(m *Mutex) +// The published caller is already safe for GC while blocked. Wait for a real +// condition signal instead of waking every goroutine on a polling timer. func condWait(cond *Cond, mutex *Mutex) c.Int { - // All hosted condition waiters must check their predicate after return. - // A timed wake lets a parked pthread acknowledge a collection. Release - // the application lock first so another mutator can also reach a safepoint. - wasiGCCondTimedWait(cond, mutex) - mutex.Unlock() - wasiGCSafepoint() - mutex.Lock() + wasiGCCondTimedWait(unsafe.Pointer(cond), unsafe.Pointer(mutex), -1, false) return 0 }