From e368e17ffdac42a6ff0e5ea22ea197a1d943c0a4 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 11:00:58 +0200 Subject: [PATCH 01/16] test: prevent failed HTTP requests from hanging Caddy tests --- caddy/caddy_test.go | 26 ++++++++--------- caddy/hotreload_test.go | 64 ++++++++++++++++------------------------- 2 files changed, 37 insertions(+), 53 deletions(-) diff --git a/caddy/caddy_test.go b/caddy/caddy_test.go index 331b13ee5d..7d821f9252 100644 --- a/caddy/caddy_test.go +++ b/caddy/caddy_test.go @@ -98,8 +98,8 @@ func TestPHP(t *testing.T) { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) - wg.Done() }(i) } wg.Wait() @@ -161,8 +161,8 @@ func TestWorker(t *testing.T) { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) - wg.Done() }(i) } wg.Wait() @@ -213,9 +213,9 @@ func TestGlobalAndModuleWorker(t *testing.T) { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse("http://localhost:"+testPort+"/worker-with-env.php", http.StatusOK, "Worker has APP_ENV=module") tester.AssertGetResponse("http://localhost:"+testPortTwo+"/worker-with-env.php", http.StatusOK, "Worker has APP_ENV=global") - wg.Done() }(i) } wg.Wait() @@ -287,9 +287,9 @@ func TestNamedModuleWorkers(t *testing.T) { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse("http://localhost:"+testPort+"/worker-with-env.php", http.StatusOK, "Worker has APP_ENV=one") tester.AssertGetResponse("http://localhost:"+testPortTwo+"/worker-with-env.php", http.StatusOK, "Worker has APP_ENV=two") - wg.Done() }(i) } wg.Wait() @@ -722,8 +722,8 @@ func TestMetrics(t *testing.T) { for i := range 10 { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) - wg.Done() }(i) } wg.Wait() @@ -798,8 +798,8 @@ func TestWorkerMetrics(t *testing.T) { for i := range 10 { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) - wg.Done() }(i) } wg.Wait() @@ -955,8 +955,8 @@ func TestNamedWorkerMetrics(t *testing.T) { for i := range 10 { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) - wg.Done() }(i) } wg.Wait() @@ -1049,8 +1049,8 @@ func TestAutoWorkerConfig(t *testing.T) { for i := range 10 { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) - wg.Done() }(i) } wg.Wait() @@ -1285,11 +1285,11 @@ func TestMaxWaitTime(t *testing.T) { wg.Add(10) for range 10 { go func() { + defer wg.Done() statusCode := getStatusCode("http://localhost:"+testPort+"/sleep.php?sleep=10", t) if statusCode == http.StatusServiceUnavailable { success.Store(true) } - wg.Done() }() } wg.Wait() @@ -1332,11 +1332,11 @@ func TestMaxWaitTimeWorker(t *testing.T) { wg.Add(10) for range 10 { go func() { + defer wg.Done() statusCode := getStatusCode("http://localhost:"+testPort+"/sleep.php?sleep=10&iteration=1", t) if statusCode == http.StatusServiceUnavailable { success.Store(true) } - wg.Done() }() } wg.Wait() @@ -1425,8 +1425,8 @@ func TestMultiWorkersMetrics(t *testing.T) { for i := range 10 { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) - wg.Done() }(i) } wg.Wait() @@ -1533,8 +1533,8 @@ func TestDisabledMetrics(t *testing.T) { for i := range 10 { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/index.php?i=%d", i), http.StatusOK, fmt.Sprintf("I am by birth a Genevese (%d)", i)) - wg.Done() }(i) } wg.Wait() @@ -1642,8 +1642,8 @@ func TestWorkerRestart(t *testing.T) { for i := range 10 { wg.Add(1) go func(i int) { + defer wg.Done() tester.AssertGetResponse(fmt.Sprintf("http://localhost:"+testPort+"/worker-restart.php?i=%d", i), http.StatusOK, fmt.Sprintf("Counter (%d)", i)) - wg.Done() }(i) } wg.Wait() diff --git a/caddy/hotreload_test.go b/caddy/hotreload_test.go index 9eed620453..a6d52656fc 100644 --- a/caddy/hotreload_test.go +++ b/caddy/hotreload_test.go @@ -9,7 +9,6 @@ import ( "os" "path/filepath" "strings" - "sync" "testing" "time" @@ -53,49 +52,34 @@ func TestHotReload(t *testing.T) { } `, "caddyfile") - var connected, received sync.WaitGroup + cx, cancel := context.WithCancel(t.Context()) + defer cancel() + req, err := http.NewRequestWithContext(cx, http.MethodGet, "http://localhost:"+testPort+u, nil) + require.NoError(t, err) + resp := tester.AssertResponseCode(req, http.StatusOK) + defer resp.Body.Close() - connected.Add(1) - received.Go(func() { - cx, cancel := context.WithCancel(t.Context()) - req, _ := http.NewRequest(http.MethodGet, "http://localhost:"+testPort+u, nil) - req = req.WithContext(cx) - resp := tester.AssertResponseCode(req, http.StatusOK) - - var receivedBody strings.Builder - - buf := make([]byte, 1024) - isConnected := false - for { - n, err := resp.Body.Read(buf) - if n > 0 { - receivedBody.Write(buf[:n]) - } - if !isConnected { - // wait for the first bytes before marking the client as connected - isConnected = true - connected.Done() - } - if strings.Contains(receivedBody.String(), "index.php") { - cancel() - - break - } - // Surface the read error only after checking the buffer: on - // Windows the SSE server sometimes flushes the event and closes - // the connection in the same syscall, so Read returns (n>0, EOF) - // and we'd otherwise fail despite having the data we wanted. - require.NoError(t, err) - } - - require.NoError(t, resp.Body.Close()) - }) - - connected.Wait() + // Wait for the first bytes before changing the file, so the subscription + // is ready. Keep reads on the test goroutine so a failed request or read + // cannot leave a readiness WaitGroup blocked forever. + buf := make([]byte, 1024) + _, err = resp.Body.Read(buf) + require.NoError(t, err) require.NoError(t, os.WriteFile(indexFile, []byte(" Date: Wed, 23 Sep 2026 11:22:26 +0200 Subject: [PATCH 02/16] test: isolate Caddy listeners between integration tests Load each test configuration directly and stop the admin endpoint and HTTP listeners during cleanup. This avoids the shared Windows listener deadline race caused by reloading the previous test server and removes unbounded HTTP diagnostic requests. Wait for listener release before reusing ports and cover that lifecycle with a regression test. --- caddy/admin_test.go | 12 ++--- caddy/caddy_test.go | 14 ++--- caddy/hotreload_test.go | 2 +- caddy/module_test.go | 2 +- caddy/testserver_test.go | 114 +++++++++++++++++++++++++++++++++++++++ caddy/watcher_test.go | 2 +- 6 files changed, 131 insertions(+), 15 deletions(-) create mode 100644 caddy/testserver_test.go diff --git a/caddy/admin_test.go b/caddy/admin_test.go index b5fe37daae..44912af6e2 100644 --- a/caddy/admin_test.go +++ b/caddy/admin_test.go @@ -21,7 +21,7 @@ import ( func TestRestartWorkerViaAdminApi(t *testing.T) { tester := caddytest.NewTester(t) - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 @@ -54,7 +54,7 @@ func TestRestartWorkerViaAdminApi(t *testing.T) { func TestShowTheCorrectThreadDebugStatus(t *testing.T) { tester := caddytest.NewTester(t) - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 @@ -89,7 +89,7 @@ func TestShowTheCorrectThreadDebugStatus(t *testing.T) { func TestThreadDebugStateMetricsAfterRequests(t *testing.T) { tester := caddytest.NewTester(t) - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 @@ -132,7 +132,7 @@ func TestAutoScaleWorkerThreads(t *testing.T) { maxTries := 10 requestsPerTry := 200 tester := caddytest.NewTester(t) - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 @@ -189,7 +189,7 @@ func TestAutoScaleRegularThreadsOnAutomaticThreadLimit(t *testing.T) { maxTries := 10 requestsPerTry := 200 tester := caddytest.NewTester(t) - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 @@ -278,7 +278,7 @@ func getNumThreads(t *testing.T, tester *caddytest.Tester) int { func TestAddModuleWorkerViaAdminApi(t *testing.T) { // Initialize a server with admin API enabled tester := caddytest.NewTester(t) - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 diff --git a/caddy/caddy_test.go b/caddy/caddy_test.go index 7d821f9252..6f1c52f4ed 100644 --- a/caddy/caddy_test.go +++ b/caddy/caddy_test.go @@ -23,12 +23,11 @@ import ( ) // initServer initializes a Caddy test server and waits for it to be ready. -// After InitServer, it polls the server to handle a race condition on macOS where -// SO_REUSEPORT can briefly route connections to the old listener being shut down, -// resulting in "connection reset by peer". +// It polls the HTTP endpoint after loading the configuration to wait for the +// listener to start accepting requests. func initServer(t *testing.T, tester *caddytest.Tester, config string, format string) { t.Helper() - tester.InitServer(config, format) + initTestServer(t, tester, config, format) client := &http.Client{Timeout: 1 * time.Second} require.Eventually(t, func() bool { @@ -65,6 +64,9 @@ func escapeMetricLabel(s string) string { } func TestMain(m *testing.M) { + // Bound helpers using http.Get, including diagnostics and metrics requests. + http.DefaultClient.Timeout = caddytest.Default.TestRequestTimeout + // setup custom environment vars for TestOsEnv if os.Setenv("ENV1", "value1") != nil || os.Setenv("ENV2", "value2") != nil { fmt.Println("Failed to set environment variables for tests") @@ -1228,7 +1230,7 @@ func TestOsEnv(t *testing.T) { require.NoError(t, listener.Close()) tester := caddytest.NewTester(t) - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 @@ -1806,7 +1808,7 @@ func TestDd(t *testing.T) { func TestOpcacheReset(t *testing.T) { tester := caddytest.NewTester(t) tester.Client.Timeout = 60 * time.Second - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 diff --git a/caddy/hotreload_test.go b/caddy/hotreload_test.go index a6d52656fc..782bb96735 100644 --- a/caddy/hotreload_test.go +++ b/caddy/hotreload_test.go @@ -29,7 +29,7 @@ func TestHotReload(t *testing.T) { // SSE roundtrip below on slow CI runners (notably emulated armv7). // 30s keeps the test bounded so a real regression fails fast. tester.Client.Timeout = 30 * time.Second - tester.InitServer(` + initTestServer(t, tester, ` { debug skip_install_trust diff --git a/caddy/module_test.go b/caddy/module_test.go index 2934cf2b9a..4d017fa785 100644 --- a/caddy/module_test.go +++ b/caddy/module_test.go @@ -16,7 +16,7 @@ func TestRootBehavesTheSameOutsideAndInsidePhpServer(t *testing.T) { expectedFileResponse, _ := os.ReadFile("../testdata/files/static.txt") hostWithRootOutside := "http://localhost:" + testPort hostWithRootInside := "http://localhost:" + testPortTwo - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 diff --git a/caddy/testserver_test.go b/caddy/testserver_test.go new file mode 100644 index 0000000000..3730d8f9d2 --- /dev/null +++ b/caddy/testserver_test.go @@ -0,0 +1,114 @@ +package caddy_test + +import ( + "encoding/json" + "net" + "net/http" + "testing" + "time" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig" + "github.com/caddyserver/caddy/v2/caddytest" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/dunglas/frankenphp" + "github.com/stretchr/testify/require" +) + +// initTestServer starts each test with its own listeners. Reloading the previous +// test's configuration can leave Caddy's shared Windows admin listener with an +// expired accept deadline, preventing all subsequent admin requests. +func initTestServer(t *testing.T, tester *caddytest.Tester, config, format string) { + t.Helper() + if testing.Short() { + t.Skip("skipping integration test in short mode") + } + + configJSON := []byte(config) + if format != "json" { + adapter := caddyconfig.GetAdapter(format) + require.NotNil(t, adapter, "unknown config adapter %q", format) + var err error + configJSON, _, err = adapter.Adapt(configJSON, nil) + require.NoError(t, err) + } + + var cfg caddy.Config + require.NoError(t, json.Unmarshal(configJSON, &cfg)) + var httpApp caddyhttp.App + require.NoError(t, json.Unmarshal(cfg.AppsRaw["http"], &httpApp)) + + var listeners []caddy.NetworkAddress + addListener := func(address string) { + t.Helper() + addr, err := caddy.ParseNetworkAddress(address) + require.NoError(t, err) + listeners = append(listeners, addr.Expand()...) + } + if cfg.Admin == nil || !cfg.Admin.Disabled { + address := caddy.DefaultAdminListen + if cfg.Admin != nil && cfg.Admin.Listen != "" { + address = cfg.Admin.Listen + } + addListener(address) + } + for _, server := range httpApp.Servers { + for _, address := range server.Listen { + addListener(address) + } + } + + t.Cleanup(func() { + tester.Client.CloseIdleConnections() + http.DefaultClient.CloseIdleConnections() + if t.Failed() { + t.Logf("Caddy configuration:\n%s", configJSON) + } + // Disable the admin endpoint through the Go API: caddy.Stop leaves it + // running, and fetching diagnostics over HTTP can hang if it is broken. + require.NoError(t, caddy.Load([]byte(`{"admin":{"disabled":true}}`), true)) + frankenphp.Shutdown() + // Caddy closes listeners asynchronously. Wait until the old sockets + // are released before the next test binds the same addresses. + require.Eventually(t, func() bool { + for _, addr := range listeners { + if caddy.ListenerUsage(addr.Network, addr.JoinHostPort(0)) != 0 { + return false + } + // The pool entry is removed before the socket is closed. + listener, err := net.Listen(addr.Network, addr.JoinHostPort(0)) + if err != nil { + return false + } + if err := listener.Close(); err != nil { + return false + } + } + return true + }, 5*time.Second, 10*time.Millisecond, "Caddy test listeners were not released") + }) + + // Loading directly avoids caddytest's bootstrap admin listener and the + // immediate reload of that listener when the real configuration is sent. + require.NoError(t, caddy.Load(configJSON, true)) +} + +func TestCaddyTestServerReleasesListeners(t *testing.T) { + for _, name := range []string{"first", "second"} { + t.Run(name, func(t *testing.T) { + tester := caddytest.NewTester(t) + initTestServer(t, tester, ` + { + admin localhost:2999 + } + http://localhost:`+testPort+` { + respond "ready" + } + `, "caddyfile") + tester.AssertGetResponse("http://localhost:"+testPort, http.StatusOK, "ready") + }) + + require.Zero(t, caddy.ListenerUsage("tcp", "localhost:2999")) + require.Zero(t, caddy.ListenerUsage("tcp", ":"+testPort)) + } +} diff --git a/caddy/watcher_test.go b/caddy/watcher_test.go index b53afae291..c3b2816cc0 100644 --- a/caddy/watcher_test.go +++ b/caddy/watcher_test.go @@ -31,7 +31,7 @@ func waitForListener(t *testing.T, addr string) { func TestWorkerWithInactiveWatcher(t *testing.T) { tester := caddytest.NewTester(t) - tester.InitServer(` + initTestServer(t, tester, ` { skip_install_trust admin localhost:2999 From 7eafaf384cebb74ac8f3902b6685e49385fda477 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 12:36:37 +0200 Subject: [PATCH 03/16] test: handle asynchronous hot-reload watcher startup The SSE subscription can become ready before the native file watcher installs its watches. Direct Caddy startup exposed this race in Linux CI, where the single file creation was missed. Retry file updates past watcher startup and report SSE read errors on the test goroutine while keeping the request deadline and cancellation. --- caddy/hotreload_test.go | 42 +++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/caddy/hotreload_test.go b/caddy/hotreload_test.go index 782bb96735..e2c572ac3b 100644 --- a/caddy/hotreload_test.go +++ b/caddy/hotreload_test.go @@ -60,23 +60,45 @@ func TestHotReload(t *testing.T) { defer resp.Body.Close() // Wait for the first bytes before changing the file, so the subscription - // is ready. Keep reads on the test goroutine so a failed request or read - // cannot leave a readiness WaitGroup blocked forever. + // is ready. A failed request or read must not leave a readiness wait blocked. buf := make([]byte, 1024) _, err = resp.Body.Read(buf) require.NoError(t, err) - require.NoError(t, os.WriteFile(indexFile, []byte(" Date: Wed, 23 Sep 2026 13:53:06 +0200 Subject: [PATCH 04/16] test: check deferred hot-reload response close --- caddy/hotreload_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/caddy/hotreload_test.go b/caddy/hotreload_test.go index e2c572ac3b..c8f802fa32 100644 --- a/caddy/hotreload_test.go +++ b/caddy/hotreload_test.go @@ -57,7 +57,7 @@ func TestHotReload(t *testing.T) { req, err := http.NewRequestWithContext(cx, http.MethodGet, "http://localhost:"+testPort+u, nil) require.NoError(t, err) resp := tester.AssertResponseCode(req, http.StatusOK) - defer resp.Body.Close() + defer func() { require.NoError(t, resp.Body.Close()) }() // Wait for the first bytes before changing the file, so the subscription // is ready. A failed request or read must not leave a readiness wait blocked. From f514372b949fc654cbe8345dd0267059e05c8e05 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 14:48:32 +0200 Subject: [PATCH 05/16] test: report extension subprocess crashes Run the native extension test in a subprocess so an abrupt Windows exit reports its status even after test output has been printed. Log initialization, signal checks, request handling, and shutdown to identify where the process exits. Keep failures fatal and bound the child test to one minute. --- internal/testext/ext_test.go | 23 +++++++++++++++++++++-- internal/testext/exttest.go | 11 ++++++++++- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/internal/testext/ext_test.go b/internal/testext/ext_test.go index 3e9cfa1436..870f1fe94b 100644 --- a/internal/testext/ext_test.go +++ b/internal/testext/ext_test.go @@ -1,7 +1,26 @@ package testext -import "testing" +import ( + "os" + "os/exec" + "testing" +) func TestRegisterExtension(t *testing.T) { - testRegisterExtension(t) + const childEnv = "FRANKENPHP_TEST_EXTENSION_CHILD" + if os.Getenv(childEnv) == "1" { + testRegisterExtension(t) + return + } + + // go test omits a Windows process's exit status once it has printed any + // output. Run the native extension test in a child so a crash reports both + // its exit status and the last reached stage instead of just "unknown". + cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.v", "-test.run=^TestRegisterExtension$", "-test.timeout=1m") + cmd.Env = append(os.Environ(), childEnv+"=1") + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("extension test subprocess failed: %v\n%s", err, output) + } + t.Logf("extension test subprocess:\n%s", output) } diff --git a/internal/testext/exttest.go b/internal/testext/exttest.go index 1704d3142d..e4940caaa6 100644 --- a/internal/testext/exttest.go +++ b/internal/testext/exttest.go @@ -48,11 +48,18 @@ func testRegisterExtension(t *testing.T) { }() defer close(stop) + t.Log("initializing PHP with the registered extensions") err := frankenphp.Init() require.Nil(t, err) - defer frankenphp.Shutdown() + defer func() { + t.Log("shutting down PHP") + frankenphp.Shutdown() + t.Log("PHP shutdown completed") + }() + t.Log("checking the signal handler") require.Equal(t, int(C.SUCCESS), int(C.test_signal_handler())) + t.Log("checking recovery from a Go nil-pointer panic") assert.Panics(t, func() { var p *int _ = *p @@ -64,9 +71,11 @@ func testRegisterExtension(t *testing.T) { req, err = frankenphp.NewRequestWithContext(req, frankenphp.WithRequestDocumentRoot("./testdata", false)) assert.NoError(t, err) + t.Log("serving the extension-list request") err = frankenphp.ServeHTTP(w, req) assert.NoError(t, err) + t.Log("checking the registered extension names") resp := w.Result() body, _ := io.ReadAll(resp.Body) assert.Contains(t, string(body), "ext1") From 027f0f11070cfddf53c19ef78549d7789c35cd79 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 15:39:04 +0200 Subject: [PATCH 06/16] ci: capture native test crash backtraces --- .github/scripts/test-with-backtrace.sh | 23 +++++++++++++++++++++++ .github/workflows/tests.yaml | 8 +++++++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/test-with-backtrace.sh diff --git a/.github/scripts/test-with-backtrace.sh b/.github/scripts/test-with-backtrace.sh new file mode 100644 index 0000000000..328b088e02 --- /dev/null +++ b/.github/scripts/test-with-backtrace.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +# Go removes its temporary test binary after it exits. Read native crash dumps +# before returning to Go, while the executable and its debug symbols still exist. +set -euo pipefail +shopt -s nullglob + +status=0 +"$@" || status=$? + +if ((status != 0)); then + for core in frankenphp-core.*; do + gdb --batch --quiet \ + -iex 'set auto-load off' \ + -ex 'set debuginfod enabled off' \ + -ex 'set pagination off' \ + -ex 'set print frame-arguments none' \ + -ex 'thread apply all bt' \ + "$1" "$core" || true + done +fi + +exit "$status" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a9154651cf..8dfbaee84f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -61,8 +61,14 @@ jobs: run: go build - name: Install gotestsum run: go install gotest.tools/gotestsum@latest + - name: Enable native crash backtraces + run: | + sudo apt-get install -y gdb + sudo sysctl -w kernel.core_pattern=frankenphp-core.%p - name: Run library tests - run: gotestsum -- -race ./... + run: | + ulimit -c unlimited + gotestsum -- -race -exec="bash $GITHUB_WORKSPACE/.github/scripts/test-with-backtrace.sh" ./... - name: Run Caddy module tests working-directory: caddy/ run: gotestsum -- -race ./... From 212e159b4688fe41a6caeeff57f384d374c3d5fd Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 16:51:06 +0200 Subject: [PATCH 07/16] ci: fix native crash diagnostic setup --- .github/scripts/test-with-backtrace.sh | 0 .github/workflows/tests.yaml | 1 + 2 files changed, 1 insertion(+) mode change 100644 => 100755 .github/scripts/test-with-backtrace.sh diff --git a/.github/scripts/test-with-backtrace.sh b/.github/scripts/test-with-backtrace.sh old mode 100644 new mode 100755 diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 8dfbaee84f..28bfba367b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -63,6 +63,7 @@ jobs: run: go install gotest.tools/gotestsum@latest - name: Enable native crash backtraces run: | + sudo apt-get update sudo apt-get install -y gdb sudo sysctl -w kernel.core_pattern=frankenphp-core.%p - name: Run library tests From 7f10e9b483c110380141f26e4cf74efc9e2ec0f6 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 17:08:39 +0200 Subject: [PATCH 08/16] test: wait for PHP thread readiness in admin checks --- caddy/admin_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/caddy/admin_test.go b/caddy/admin_test.go index 44912af6e2..e8ccf6cff1 100644 --- a/caddy/admin_test.go +++ b/caddy/admin_test.go @@ -77,14 +77,20 @@ func TestShowTheCorrectThreadDebugStatus(t *testing.T) { } `, "caddyfile") + // Regular threads finish their startup transition asynchronously. + require.Eventually(t, func() bool { + threads := frankenphp.DebugState().ThreadDebugStates + return len(threads) == 3 && threads[0].State == "ready" + }, 5*time.Second, 10*time.Millisecond, "regular PHP thread did not become ready") + debugState := getDebugState(t, tester) // assert that the correct threads are present in the thread info - assert.Equal(t, debugState.ThreadDebugStates[0].State, "ready") + require.Len(t, debugState.ThreadDebugStates, 3) + assert.Equal(t, "ready", debugState.ThreadDebugStates[0].State) assert.Contains(t, debugState.ThreadDebugStates[1].Name, "worker-with-counter.php") assert.Contains(t, debugState.ThreadDebugStates[2].Name, "index.php") - assert.Equal(t, debugState.ReservedThreadCount, 3) - assert.Len(t, debugState.ThreadDebugStates, 3) + assert.Equal(t, 3, debugState.ReservedThreadCount) } func TestThreadDebugStateMetricsAfterRequests(t *testing.T) { From 85a2ac2be499ec9e689ac2efc56c28fd84ab3d4f Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 18:42:05 +0200 Subject: [PATCH 09/16] test: remove temporary crash diagnostics --- .github/scripts/test-with-backtrace.sh | 23 ----------------------- .github/workflows/tests.yaml | 9 +-------- caddy/admin_test.go | 6 +++--- internal/testext/ext_test.go | 23 ++--------------------- internal/testext/exttest.go | 11 +---------- 5 files changed, 7 insertions(+), 65 deletions(-) delete mode 100755 .github/scripts/test-with-backtrace.sh diff --git a/.github/scripts/test-with-backtrace.sh b/.github/scripts/test-with-backtrace.sh deleted file mode 100755 index 328b088e02..0000000000 --- a/.github/scripts/test-with-backtrace.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -# Go removes its temporary test binary after it exits. Read native crash dumps -# before returning to Go, while the executable and its debug symbols still exist. -set -euo pipefail -shopt -s nullglob - -status=0 -"$@" || status=$? - -if ((status != 0)); then - for core in frankenphp-core.*; do - gdb --batch --quiet \ - -iex 'set auto-load off' \ - -ex 'set debuginfod enabled off' \ - -ex 'set pagination off' \ - -ex 'set print frame-arguments none' \ - -ex 'thread apply all bt' \ - "$1" "$core" || true - done -fi - -exit "$status" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 28bfba367b..a9154651cf 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -61,15 +61,8 @@ jobs: run: go build - name: Install gotestsum run: go install gotest.tools/gotestsum@latest - - name: Enable native crash backtraces - run: | - sudo apt-get update - sudo apt-get install -y gdb - sudo sysctl -w kernel.core_pattern=frankenphp-core.%p - name: Run library tests - run: | - ulimit -c unlimited - gotestsum -- -race -exec="bash $GITHUB_WORKSPACE/.github/scripts/test-with-backtrace.sh" ./... + run: gotestsum -- -race ./... - name: Run Caddy module tests working-directory: caddy/ run: gotestsum -- -race ./... diff --git a/caddy/admin_test.go b/caddy/admin_test.go index e8ccf6cff1..d44deaa8d6 100644 --- a/caddy/admin_test.go +++ b/caddy/admin_test.go @@ -86,11 +86,11 @@ func TestShowTheCorrectThreadDebugStatus(t *testing.T) { debugState := getDebugState(t, tester) // assert that the correct threads are present in the thread info - require.Len(t, debugState.ThreadDebugStates, 3) - assert.Equal(t, "ready", debugState.ThreadDebugStates[0].State) + assert.Equal(t, debugState.ThreadDebugStates[0].State, "ready") assert.Contains(t, debugState.ThreadDebugStates[1].Name, "worker-with-counter.php") assert.Contains(t, debugState.ThreadDebugStates[2].Name, "index.php") - assert.Equal(t, 3, debugState.ReservedThreadCount) + assert.Equal(t, debugState.ReservedThreadCount, 3) + assert.Len(t, debugState.ThreadDebugStates, 3) } func TestThreadDebugStateMetricsAfterRequests(t *testing.T) { diff --git a/internal/testext/ext_test.go b/internal/testext/ext_test.go index 870f1fe94b..3e9cfa1436 100644 --- a/internal/testext/ext_test.go +++ b/internal/testext/ext_test.go @@ -1,26 +1,7 @@ package testext -import ( - "os" - "os/exec" - "testing" -) +import "testing" func TestRegisterExtension(t *testing.T) { - const childEnv = "FRANKENPHP_TEST_EXTENSION_CHILD" - if os.Getenv(childEnv) == "1" { - testRegisterExtension(t) - return - } - - // go test omits a Windows process's exit status once it has printed any - // output. Run the native extension test in a child so a crash reports both - // its exit status and the last reached stage instead of just "unknown". - cmd := exec.CommandContext(t.Context(), os.Args[0], "-test.v", "-test.run=^TestRegisterExtension$", "-test.timeout=1m") - cmd.Env = append(os.Environ(), childEnv+"=1") - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("extension test subprocess failed: %v\n%s", err, output) - } - t.Logf("extension test subprocess:\n%s", output) + testRegisterExtension(t) } diff --git a/internal/testext/exttest.go b/internal/testext/exttest.go index e4940caaa6..1704d3142d 100644 --- a/internal/testext/exttest.go +++ b/internal/testext/exttest.go @@ -48,18 +48,11 @@ func testRegisterExtension(t *testing.T) { }() defer close(stop) - t.Log("initializing PHP with the registered extensions") err := frankenphp.Init() require.Nil(t, err) - defer func() { - t.Log("shutting down PHP") - frankenphp.Shutdown() - t.Log("PHP shutdown completed") - }() + defer frankenphp.Shutdown() - t.Log("checking the signal handler") require.Equal(t, int(C.SUCCESS), int(C.test_signal_handler())) - t.Log("checking recovery from a Go nil-pointer panic") assert.Panics(t, func() { var p *int _ = *p @@ -71,11 +64,9 @@ func testRegisterExtension(t *testing.T) { req, err = frankenphp.NewRequestWithContext(req, frankenphp.WithRequestDocumentRoot("./testdata", false)) assert.NoError(t, err) - t.Log("serving the extension-list request") err = frankenphp.ServeHTTP(w, req) assert.NoError(t, err) - t.Log("checking the registered extension names") resp := w.Result() body, _ := io.ReadAll(resp.Body) assert.Contains(t, string(body), "ext1") From 7496bb2072b050ee5a51b84d40e98ca8d402a44f Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 19:00:59 +0200 Subject: [PATCH 10/16] test: remove redundant listener cleanup test --- caddy/testserver_test.go | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/caddy/testserver_test.go b/caddy/testserver_test.go index 3730d8f9d2..58f9d3dfde 100644 --- a/caddy/testserver_test.go +++ b/caddy/testserver_test.go @@ -92,23 +92,3 @@ func initTestServer(t *testing.T, tester *caddytest.Tester, config, format strin // immediate reload of that listener when the real configuration is sent. require.NoError(t, caddy.Load(configJSON, true)) } - -func TestCaddyTestServerReleasesListeners(t *testing.T) { - for _, name := range []string{"first", "second"} { - t.Run(name, func(t *testing.T) { - tester := caddytest.NewTester(t) - initTestServer(t, tester, ` - { - admin localhost:2999 - } - http://localhost:`+testPort+` { - respond "ready" - } - `, "caddyfile") - tester.AssertGetResponse("http://localhost:"+testPort, http.StatusOK, "ready") - }) - - require.Zero(t, caddy.ListenerUsage("tcp", "localhost:2999")) - require.Zero(t, caddy.ListenerUsage("tcp", ":"+testPort)) - } -} From 760569a3c87e253d5aa396ec2747635ae7ede836 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 19:20:06 +0200 Subject: [PATCH 11/16] ci: isolate intermittent PHP 8.5 startup crash --- .github/workflows/tests.yaml | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a9154651cf..9c33f5089f 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -61,6 +61,52 @@ jobs: run: go build - name: Install gotestsum run: go install gotest.tools/gotestsum@latest + - name: Diagnose extension startup crash + if: matrix.php-versions == '8.5' + run: | + command -v gdb + sudo sysctl -w kernel.core_pattern="${RUNNER_TEMP}/frankenphp-core.%p" + ulimit -c unlimited + go test -race -c -o "${RUNNER_TEMP}/testext.test" ./internal/testext + python3 - <<'PY' + import concurrent.futures + import os + import pathlib + import subprocess + import sys + import threading + + root = pathlib.Path(os.environ["RUNNER_TEMP"]) + binary = root / "testext.test" + env = dict(os.environ, GORACE="atexit_sleep_ms=0") + failed = threading.Event() + + def run(index): + if failed.is_set(): + return None + result = subprocess.run( + [binary, "-test.v", "-test.timeout=1m"], + cwd="internal/testext", env=env, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, + ) + if result.returncode: + failed.set() + print(f"Process {index} exited {result.returncode}:\n{result.stdout}", flush=True) + return result.returncode + + with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(run, range(300))) + attempted = [code for code in results if code is not None] + print(f"Extension startup: {sum(code == 0 for code in attempted)}/{len(attempted)} passed", flush=True) + for core in root.glob("frankenphp-core.*"): + subprocess.run([ + "gdb", "--batch", "--quiet", "-iex", "set auto-load off", + "-ex", "set debuginfod enabled off", + "-ex", "set pagination off", "-ex", "thread apply all bt 30", + binary, core, + ]) + sys.exit(any(results)) + PY - name: Run library tests run: gotestsum -- -race ./... - name: Run Caddy module tests From 8ccc807c4b2bf3737202470484dca4cd58f26b5d Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 19:21:57 +0200 Subject: [PATCH 12/16] ci: install debugger for startup crash capture --- .github/workflows/tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9c33f5089f..ec3a910367 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -64,7 +64,8 @@ jobs: - name: Diagnose extension startup crash if: matrix.php-versions == '8.5' run: | - command -v gdb + sudo apt-get -o Acquire::Retries=3 update + sudo apt-get -o Acquire::Retries=3 install -y gdb sudo sysctl -w kernel.core_pattern="${RUNNER_TEMP}/frankenphp-core.%p" ulimit -c unlimited go test -race -c -o "${RUNNER_TEMP}/testext.test" ./internal/testext From dc74c27e899bb1e058dc9b1ecc6dbeb477e33313 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 19:28:26 +0200 Subject: [PATCH 13/16] ci: capture allocator failure cores during startup --- .github/workflows/tests.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index ec3a910367..f6d7fdee64 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -79,7 +79,7 @@ jobs: root = pathlib.Path(os.environ["RUNNER_TEMP"]) binary = root / "testext.test" - env = dict(os.environ, GORACE="atexit_sleep_ms=0") + env = dict(os.environ, GORACE="atexit_sleep_ms=0", GOTRACEBACK="crash") failed = threading.Event() def run(index): @@ -104,6 +104,7 @@ jobs: "gdb", "--batch", "--quiet", "-iex", "set auto-load off", "-ex", "set debuginfod enabled off", "-ex", "set pagination off", "-ex", "thread apply all bt 30", + "-ex", "info sharedlibrary", "-ex", "p 'runtime.mheap_'.sweepgen", binary, core, ]) sys.exit(any(results)) From d631d3fce306e8cb4e7afec74428bb41b4b5fa27 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 19:33:17 +0200 Subject: [PATCH 14/16] ci: recover original fault context from signal wrapper --- .github/workflows/tests.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f6d7fdee64..6f0ef717cd 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -105,6 +105,13 @@ jobs: "-ex", "set debuginfod enabled off", "-ex", "set pagination off", "-ex", "thread apply all bt 30", "-ex", "info sharedlibrary", "-ex", "p 'runtime.mheap_'.sweepgen", + "-ex", "frame 3", "-ex", "set language c", + "-ex", "set $uc = (char *)ctx", "-ex", "x/23gx $uc+40", + "-ex", "set $pc = *(unsigned long *)($uc+168)", + "-ex", "set $sp = *(unsigned long *)($uc+160)", + "-ex", "set $rbp = *(unsigned long *)($uc+120)", + "-ex", "set $r14 = *(unsigned long *)($uc+88)", + "-ex", "x/i $pc", "-ex", "bt 30", binary, core, ]) sys.exit(any(results)) From b81c7b099137f524d9017c0e5a731d7f4f8f15d7 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 19:44:07 +0200 Subject: [PATCH 15/16] ci: disable JIT during threaded test startup --- .github/actions/setup-php/action.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/setup-php/action.yaml b/.github/actions/setup-php/action.yaml index 9bc8c49208..6227376e84 100644 --- a/.github/actions/setup-php/action.yaml +++ b/.github/actions/setup-php/action.yaml @@ -12,6 +12,8 @@ runs: with: php-version: ${{ inputs.php-version }} ini-file: development + # JIT's fixed-address mappings can race Go thread creation during startup. + ini-values: opcache.jit=disable coverage: none tools: none env: From 1bf8fc90c156f91189dbe3eb600dfeacf144a300 Mon Sep 17 00:00:00 2001 From: henderkes Date: Wed, 23 Sep 2026 19:57:12 +0200 Subject: [PATCH 16/16] ci: remove temporary startup crash diagnostics --- .github/workflows/tests.yaml | 55 ------------------------------------ 1 file changed, 55 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 6f0ef717cd..a9154651cf 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -61,61 +61,6 @@ jobs: run: go build - name: Install gotestsum run: go install gotest.tools/gotestsum@latest - - name: Diagnose extension startup crash - if: matrix.php-versions == '8.5' - run: | - sudo apt-get -o Acquire::Retries=3 update - sudo apt-get -o Acquire::Retries=3 install -y gdb - sudo sysctl -w kernel.core_pattern="${RUNNER_TEMP}/frankenphp-core.%p" - ulimit -c unlimited - go test -race -c -o "${RUNNER_TEMP}/testext.test" ./internal/testext - python3 - <<'PY' - import concurrent.futures - import os - import pathlib - import subprocess - import sys - import threading - - root = pathlib.Path(os.environ["RUNNER_TEMP"]) - binary = root / "testext.test" - env = dict(os.environ, GORACE="atexit_sleep_ms=0", GOTRACEBACK="crash") - failed = threading.Event() - - def run(index): - if failed.is_set(): - return None - result = subprocess.run( - [binary, "-test.v", "-test.timeout=1m"], - cwd="internal/testext", env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - ) - if result.returncode: - failed.set() - print(f"Process {index} exited {result.returncode}:\n{result.stdout}", flush=True) - return result.returncode - - with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool: - results = list(pool.map(run, range(300))) - attempted = [code for code in results if code is not None] - print(f"Extension startup: {sum(code == 0 for code in attempted)}/{len(attempted)} passed", flush=True) - for core in root.glob("frankenphp-core.*"): - subprocess.run([ - "gdb", "--batch", "--quiet", "-iex", "set auto-load off", - "-ex", "set debuginfod enabled off", - "-ex", "set pagination off", "-ex", "thread apply all bt 30", - "-ex", "info sharedlibrary", "-ex", "p 'runtime.mheap_'.sweepgen", - "-ex", "frame 3", "-ex", "set language c", - "-ex", "set $uc = (char *)ctx", "-ex", "x/23gx $uc+40", - "-ex", "set $pc = *(unsigned long *)($uc+168)", - "-ex", "set $sp = *(unsigned long *)($uc+160)", - "-ex", "set $rbp = *(unsigned long *)($uc+120)", - "-ex", "set $r14 = *(unsigned long *)($uc+88)", - "-ex", "x/i $pc", "-ex", "bt 30", - binary, core, - ]) - sys.exit(any(results)) - PY - name: Run library tests run: gotestsum -- -race ./... - name: Run Caddy module tests