diff --git a/bgworker_test.go b/bgworker_test.go new file mode 100644 index 0000000000..2a99f9448d --- /dev/null +++ b/bgworker_test.go @@ -0,0 +1,664 @@ +package frankenphp_test + +import ( + "bytes" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/dunglas/frankenphp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// requireFileEventually asserts that `path` appears on disk before the +// deadline. Wraps require.Eventually so call sites stay short. +func requireFileEventually(t testing.TB, path string, msgAndArgs ...any) { + t.Helper() + require.Eventually(t, func() bool { + _, err := os.Stat(path) + return err == nil + }, 5*time.Second, 25*time.Millisecond, msgAndArgs...) +} + +// requireFileContentEventually waits for `path` to appear with content and +// returns it +func requireFileContentEventually(t *testing.T, path string) string { + t.Helper() + require.Eventually(t, func() bool { + b, err := os.ReadFile(path) + return err == nil && len(b) > 0 + }, 5*time.Second, 25*time.Millisecond, "file %q did not appear", path) + b, err := os.ReadFile(path) + require.NoError(t, err) + + return string(b) +} + +// TestBackgroundWorkerLifecycle boots a background worker that touches a +// sentinel file then parks on its handle. It proves the bg worker runs +// (sentinel appears) and that Shutdown returns within a reasonable time. +// The test asserts on Shutdown timing, so it manages Shutdown itself +// instead of using initServers' t.Cleanup hook. +func TestBackgroundWorkerLifecycle(t *testing.T) { + tmp := t.TempDir() + sentinel := filepath.Join(tmp, "bg-lifecycle.sentinel") + + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-lifecycle", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + + requireFileEventually(t, sentinel, "background worker did not touch sentinel") + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatalf("Shutdown did not return within 10s") + } +} + +// TestBackgroundWorkerCrashRestarts boots a worker that exit(1)s on its +// first run and touches a "restarted" sentinel on its second run. The +// sentinel proves the crash-restart loop fired. +func TestBackgroundWorkerCrashRestarts(t *testing.T) { + tmp := t.TempDir() + crashMarker := filepath.Join(tmp, "bg-crash.marker") + restarted := filepath.Join(tmp, "bg-crash.restarted") + + initServers(t, + frankenphp.WithWorkers("bg-crash", "testdata/bgworker/crash.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{ + "BG_CRASH_MARKER": crashMarker, + "BG_RESTARTED_SENTINEL": restarted, + }), + ), + frankenphp.WithNumThreads(2), + ) + + requireFileEventually(t, restarted, "background worker did not restart after crash") +} + +// TestBackgroundWorkerOnServer scopes a background worker to a Server. It +// proves that the worker inherits the server env (the sentinel directory is +// declared on the server, not on the worker), that FRANKENPHP_WORKER_BACKGROUND +// holds the worker name, and that the worker does not intercept HTTP requests +// served by the same server. +func TestBackgroundWorkerOnServer(t *testing.T) { + tmp := t.TempDir() + + server, err := frankenphp.NewServer( + testDataDir, + frankenphp.WithServerName("sidekick-server"), + frankenphp.WithServerEnv(map[string]string{"BG_SENTINEL_DIR": tmp}), + ) + require.NoError(t, err) + + globalSentinel := filepath.Join(tmp, "global.sentinel") + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("jobs", "testdata/bgworker/named.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + ), + // a global worker may reuse the name: names are scoped to their server + frankenphp.WithWorkers("jobs", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": globalSentinel}), + ), + frankenphp.WithNumThreads(3), + ) + + // named.php touches "/": the script sees + // the declared name, not the server-qualified one used by metrics and logs + requireFileEventually(t, filepath.Join(tmp, "jobs"), "background worker did not touch its per-name sentinel") + requireFileEventually(t, globalSentinel, "the global worker sharing the name did not start") + + body := serverGet(t, server, "http://example.com/index.php") + assert.Contains(t, body, "I am by birth a Genevese", "the server must still serve regular requests") +} + +// TestBackgroundWorkerValidation covers the declaration-time errors. +func TestBackgroundWorkerValidation(t *testing.T) { + t.Cleanup(frankenphp.Shutdown) + + t.Run("name is required", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("", "testdata/bgworker/basic.php", 1, frankenphp.WithWorkerBackground()), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "must have an explicit name") + }) + + t.Run("num must be >= 1", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-zero", "testdata/bgworker/basic.php", 0, frankenphp.WithWorkerBackground()), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "must declare num >= 1") + }) + + t.Run("names are unique within a server", func(t *testing.T) { + // a global and a server-scoped worker may share a name (see + // TestBackgroundWorkerOnServer), two workers of one server may not + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + err = frankenphp.Init( + frankenphp.WithServer(server), + frankenphp.WithWorkers("bg-shared", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + ), + frankenphp.WithWorkers("bg-shared", "testdata/bgworker/named.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + ), + frankenphp.WithNumThreads(3), + ) + require.ErrorContains(t, err, "two workers in a server cannot have the same name") + }) + + t.Run("early return without the handle fails startup", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-early", "testdata/bgworker/early-return.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxFailures(2), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "without calling frankenphp_worker_tick()") + }) + + t.Run("fetching the handle without ticking fails startup", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-no-tick", "testdata/bgworker/fetch-no-tick.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxFailures(2), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "without calling frankenphp_worker_tick()") + }) + + t.Run("max_threads is rejected", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-scaled", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxThreads(2), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "cannot set max_threads") + }) + + t.Run("two workers cannot report under the same name", func(t *testing.T) { + // scoping keeps names apart, except for a global name shaped like + // the ":" of a scoped one + server, err := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("api")) + require.NoError(t, err) + err = frankenphp.Init( + frankenphp.WithServer(server), + frankenphp.WithWorkers("jobs", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + ), + frankenphp.WithWorkers("api:jobs", "testdata/bgworker/named.php", 1, + frankenphp.WithWorkerBackground(), + ), + frankenphp.WithNumThreads(3), + ) + require.ErrorContains(t, err, `two workers cannot report under the same name: "api:jobs"`) + }) + + t.Run("an unregistered server scope is rejected", func(t *testing.T) { + unregistered, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + err = frankenphp.Init( + frankenphp.WithWorkers("bg-orphan", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(unregistered), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "not passed to WithServer()") + }) + + t.Run("request matchers are rejected", func(t *testing.T) { + err := frankenphp.Init( + frankenphp.WithWorkers("bg-matched", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMatcher(func(*http.Request) bool { return true }), + ), + frankenphp.WithNumThreads(2), + ) + require.ErrorContains(t, err, "cannot match requests") + }) +} + +// TestBackgroundWorkerCannotHandleRequests checks that a request targeting a +// background worker by name is refused rather than dispatched to it. +func TestBackgroundWorkerCannotHandleRequests(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("jobs", "testdata/bgworker/basic.php", 1, frankenphp.WithWorkerBackground(), frankenphp.WithWorkerServerScope(server)), + frankenphp.WithNumThreads(2), + ) + + err = server.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "http://example.com/index.php", nil), frankenphp.WithWorkerName("jobs")) + require.ErrorContains(t, err, `background worker "jobs" cannot handle requests`) +} + +// TestBackgroundWorkerParksOnRead checks that a blocking read on the handle +// is a wait too: Init() returns only once the worker is ready, and the EOF +// of the drain unblocks the read so Shutdown() returns promptly. +func TestBackgroundWorkerParksOnRead(t *testing.T) { + tmp := t.TempDir() + sentinel := filepath.Join(tmp, "bg-read.sentinel") + + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-read", "testdata/bgworker/read.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + requireFileEventually(t, sentinel, "background worker parked on a read did not start") + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not return within 10s: the read did not observe EOF") + } +} + +// TestBackgroundWorkerParksOnReceive checks that a blocking receive counts +// as a wait as well: it reaches the stream through the transport API rather +// than the read op, so Init() would hang if only reads reported readiness. +func TestBackgroundWorkerParksOnReceive(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "bg-recv.sentinel") + + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-recv", "testdata/bgworker/recv.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + requireFileEventually(t, sentinel, "background worker parked on a receive did not start") + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not return within 10s: the receive did not observe EOF") + } +} + +// TestBackgroundWorkerRestartDrainsParkedScript checks that RestartWorkers() +// wakes a parked background script through the drain and re-runs it. +func TestBackgroundWorkerRestartDrainsParkedScript(t *testing.T) { + tmp := t.TempDir() + countFile := filepath.Join(tmp, "bg-count.log") + + initServers(t, + frankenphp.WithWorkers("bg-count", "testdata/bgworker/count.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + ) + runs := func() int { + b, _ := os.ReadFile(countFile) + return bytes.Count(b, []byte("\n")) + } + require.Eventually(t, func() bool { return runs() == 1 }, 5*time.Second, 25*time.Millisecond, "background worker did not start") + + frankenphp.RestartWorkers() + + require.Eventually(t, func() bool { return runs() == 2 }, 5*time.Second, 25*time.Millisecond, "background worker was not re-run after the restart") +} + +// TestGetWorkerHandleOutsideBackgroundWorker checks the function throws on a +// regular request thread instead of handing out a stream. +func TestGetWorkerHandleOutsideBackgroundWorker(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), frankenphp.WithNumThreads(1)) + + body := serverGet(t, server, "http://example.com/handle-outside.php") + + assert.Contains(t, body, "frankenphp_get_worker_handle() can only be called from a background worker") + assert.Contains(t, body, "frankenphp_worker_tick() can only be called from a background worker") +} + +// TestBackgroundWorkerLoopTicksOnItsOwn checks the wake-up sent at start: a +// script that only ticks when its handle is readable, the shape of a script +// driven by an event loop, becomes ready without an explicit first call. +// Without the wake-up, Init() would wait for that call until the drain. +func TestBackgroundWorkerLoopTicksOnItsOwn(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "loop.sentinel") + initServers(t, + frankenphp.WithWorkers("bg-loop", "testdata/bgworker/loop.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + ) + requireFileEventually(t, sentinel, "background worker did not start") +} + +// TestBackgroundWorkerTick checks the contract of frankenphp_worker_tick(): +// true while the worker runs, false once it is drained, and still false on +// the next call +func TestBackgroundWorkerTick(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "ticks.txt") + + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-tick", "testdata/bgworker/tick.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + assert.Equal(t, "true true", requireFileContentEventually(t, sentinel)) + + // Shutdown() waits for the script to leave, so the file is final after it + frankenphp.Shutdown() + b, err := os.ReadFile(sentinel) + require.NoError(t, err) + assert.Equal(t, "true true false false", string(b)) +} + +// TestWorkerNameInServerVars checks that an HTTP worker sees FRANKENPHP_WORKER +// as it always did, and that a background worker sees its declared name in +// FRANKENPHP_WORKER_BACKGROUND and no FRANKENPHP_WORKER. +func TestWorkerNameInServerVars(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "flag.txt") + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("web", testDataDir+"worker-name.php", 1, frankenphp.WithWorkerServerScope(server)), + frankenphp.WithWorkers("jobs", "testdata/bgworker/flag.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(3), + ) + + assert.Equal(t, "1 http", serverGet(t, server, "http://example.com/worker-name.php")) + + flag := requireFileContentEventually(t, sentinel) + assert.Contains(t, flag, "'worker' => 'unset'") + assert.Contains(t, flag, "'background' => 'jobs'") +} + +// TestBackgroundWorkerPool checks that num > 1 threads share the name, each +// parks on its own handle, and one drain wakes them all. +func TestBackgroundWorkerPool(t *testing.T) { + dir := t.TempDir() + initServers(t, + frankenphp.WithWorkers("pool", "testdata/bgworker/pool.php", 3, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL_DIR": dir}), + ), + frankenphp.WithNumThreads(4), + ) + + require.Eventually(t, func() bool { + entries, _ := os.ReadDir(dir) + return len(entries) == 3 + }, 5*time.Second, 25*time.Millisecond, "the three pool threads did not all start") + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not drain the whole pool within 10s") + } +} + +// TestBackgroundWorkerMultiEntrypoint checks that two named background +// workers of one server may share a script, since they are not matched by path. +func TestBackgroundWorkerMultiEntrypoint(t *testing.T) { + tmp := t.TempDir() + first, second := filepath.Join(tmp, "first"), filepath.Join(tmp, "second") + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("first", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": first}), + ), + frankenphp.WithWorkers("second", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerServerScope(server), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": second}), + ), + frankenphp.WithNumThreads(3), + ) + + requireFileEventually(t, first, "the first worker on the shared script did not start") + requireFileEventually(t, second, "the second worker on the shared script did not start") +} + +// TestBackgroundWorkerThreadsComeOnTop checks that background threads are +// reserved on top of num_threads: one HTTP thread plus one background worker +// starts with num_threads 1. +func TestBackgroundWorkerThreadsComeOnTop(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "bg-only.sentinel") + initServers(t, + frankenphp.WithWorkers("bg-only", "testdata/bgworker/basic.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(1), + ) + + requireFileEventually(t, sentinel, "background worker did not start with num_threads 1") +} + +// TestBackgroundWorkerThreadsComeOnTopOfAutoMaxThreads checks that the +// automatic limit is an HTTP one too: the reservation is added to what it +// resolves, instead of eating into it +func TestBackgroundWorkerThreadsComeOnTopOfAutoMaxThreads(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "bg-auto.sentinel") + initServers(t, + frankenphp.WithWorkers("bg-auto", "testdata/bgworker/basic.php", 2, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + frankenphp.WithMaxThreads(-1), + frankenphp.WithPhpIni(map[string]string{"memory_limit": "-1"}), + ) + requireFileEventually(t, sentinel, "background worker did not start") + + // an unlimited memory_limit falls back to twice the HTTP threads, so + // 2*2 HTTP plus the 2 reserved, not (2+2)*2 + state := frankenphp.DebugState() + assert.Equal(t, 6, len(state.ThreadDebugStates)+state.ReservedThreadCount) +} + +// TestBackgroundWorkerBootstrapIsBounded checks that max_execution_time +// applies until the first frankenphp_worker_tick(): a setup that outlives +// it is ended as a boot failure, which fails Init() past the cap. The limit +// itself is PHP's, so the test only runs where its timers are known to +// fire under FrankenPHP, the max execution timers of ZTS builds on Linux. +func TestBackgroundWorkerBootstrapIsBounded(t *testing.T) { + if !frankenphp.Config().ZendMaxExecutionTimers { + t.Skip("max_execution_time is only reliable with Zend max execution timers") + } + + countFile := filepath.Join(t.TempDir(), "boots") + err := frankenphp.Init( + frankenphp.WithWorkers("bg-slow", "testdata/bgworker/slow-boot.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxFailures(0), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + frankenphp.WithPhpIni(map[string]string{"max_execution_time": "1"}), + ) + if err == nil { + frankenphp.Shutdown() + } + require.ErrorContains(t, err, "keeps crashing") + + b, _ := os.ReadFile(countFile) + assert.Equal(t, 1, bytes.Count(b, []byte("\n")), "the limit should have ended the one and only boot") +} + +// TestBackgroundWorkerParkingIsNotInterrupted checks that a script parked +// on its handle, past its first tick, is not cut short by the two limits it +// never disables itself: max_execution_time, which the first tick disarms +// after php_execute_script() re-armed it from the ini, and +// default_socket_timeout, which the handle overrides with an infinite read +// timeout. +func TestBackgroundWorkerParkingIsNotInterrupted(t *testing.T) { + countFile := filepath.Join(t.TempDir(), "runs") + initServers(t, + frankenphp.WithWorkers("bg-timeout", "testdata/bgworker/no-time-limit.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + frankenphp.WithPhpIni(map[string]string{"max_execution_time": "1", "max_input_time": "1", "default_socket_timeout": "1"}), + ) + + runs := func() int { + b, _ := os.ReadFile(countFile) + + return bytes.Count(b, []byte("\n")) + } + require.Eventually(t, func() bool { return runs() == 1 }, 5*time.Second, 25*time.Millisecond, "background worker did not start") + // well past both limits + time.Sleep(2500 * time.Millisecond) + assert.Equal(t, 1, runs(), "the worker was restarted, so a limit interrupted its park") +} + +// TestBackgroundWorkerHandleClosedAndFetchedAgain checks the handle cache: +// a run gets one stream, closing it yields a fresh one on the next fetch, +// and the drain still reaches the script through it. +func TestBackgroundWorkerHandleClosedAndFetchedAgain(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "refetch.txt") + + require.NoError(t, frankenphp.Init( + frankenphp.WithWorkers("bg-refetch", "testdata/bgworker/refetch.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + )) + assert.Equal(t, "same then fresh", requireFileContentEventually(t, sentinel)) + + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not return within 10s: the re-fetched handle missed the drain") + } +} + +// TestBackgroundWorkerBootFailuresThenSucceeds checks that boot failures below +// max_consecutive_failures are retried with the backoff and Init() still +// succeeds once a run reaches its ready point. +func TestBackgroundWorkerBootFailuresThenSucceeds(t *testing.T) { + tmp := t.TempDir() + countFile, sentinel := filepath.Join(tmp, "boots"), filepath.Join(tmp, "ready") + initServers(t, + frankenphp.WithWorkers("bg-flaky", "testdata/bgworker/fail-then-succeed.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile, "BG_SENTINEL": sentinel, "BG_FAIL_UNTIL": "2"}), + ), + frankenphp.WithNumThreads(2), + ) + + requireFileEventually(t, sentinel, "background worker did not recover from its boot failures") + boots, err := os.ReadFile(countFile) + require.NoError(t, err) + assert.Equal(t, "3", string(boots), "two boot failures then a success") +} + +// TestBackgroundWorkerCrashAfterReadyRestarts checks that a crash after the +// ready point restarts right away without counting toward +// max_consecutive_failures, and that a zero-timeout stream_select() counts +// as the wait. +func TestBackgroundWorkerCrashAfterReadyRestarts(t *testing.T) { + countFile := filepath.Join(t.TempDir(), "runs") + initServers(t, + frankenphp.WithWorkers("bg-crashy", "testdata/bgworker/crash-after-ready.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerMaxFailures(2), + frankenphp.WithWorkerEnv(map[string]string{"BG_COUNT_FILE": countFile}), + ), + frankenphp.WithNumThreads(2), + ) + + require.Eventually(t, func() bool { + b, _ := os.ReadFile(countFile) + return bytes.Count(b, []byte("\n")) >= 4 + }, 5*time.Second, 25*time.Millisecond, "the worker was not restarted after crashing past its ready point") +} + +// TestBackgroundWorkerRebootForceKillsStuckScript checks that a script +// ignoring its handle does not stall RestartWorkers() past the reboot grace +// period: the force-kill ends it and the next run parks normally. +func TestBackgroundWorkerRebootForceKillsStuckScript(t *testing.T) { + if runtime.GOOS != "linux" && runtime.GOOS != "freebsd" { + t.Skipf("force-kill cannot interrupt a blocking syscall on %s", runtime.GOOS) + } + + tmp := t.TempDir() + once, sentinel := filepath.Join(tmp, "once"), filepath.Join(tmp, "parked") + initServers(t, + frankenphp.WithWorkers("bg-stuck", "testdata/bgworker/stuck.php", 1, + frankenphp.WithWorkerBackground(), + frankenphp.WithWorkerEnv(map[string]string{"BG_ONCE": once, "BG_SENTINEL": sentinel}), + ), + frankenphp.WithNumThreads(2), + ) + requireFileEventually(t, once, "background worker never entered its sleep") + + start := time.Now() + frankenphp.RestartWorkers() + assert.WithinDuration(t, start, time.Now(), 10*time.Second, "the reboot must force-kill the stuck script within its grace period") + + requireFileEventually(t, sentinel, "the re-run script did not park") +} diff --git a/caddy/app.go b/caddy/app.go index fcee129180..88636401c4 100644 --- a/caddy/app.go +++ b/caddy/app.go @@ -17,7 +17,6 @@ import ( "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" "github.com/dunglas/frankenphp" - "github.com/dunglas/frankenphp/internal/fastabs" ) var ( @@ -60,15 +59,14 @@ type FrankenPHPApp struct { // EXPERIMENTAL: MaxRequests sets the maximum number of requests a PHP thread handles before restarting (0 = unlimited) MaxRequests int `json:"max_requests,omitempty"` - opts []frankenphp.Option - metrics frankenphp.Metrics - ctx context.Context - logger *slog.Logger - modules []*FrankenPHPModule - usedWorkerNames map[string]bool - httpApp *caddyhttp.App - hasStarted atomic.Bool - started chan any + opts []frankenphp.Option + metrics frankenphp.Metrics + ctx context.Context + logger *slog.Logger + modules []*FrankenPHPModule + httpApp *caddyhttp.App + hasStarted atomic.Bool + started chan any } var errIni = errors.New(`"php_ini" must be in the format: php_ini "" ""`) @@ -133,7 +131,6 @@ func (f *FrankenPHPApp) Start() error { // register global workers for _, w := range f.Workers { w.FileName = repl.ReplaceKnown(w.FileName, "") - w.Name = f.createUniqueWorkerName(w, "") opts, err := w.toWorkerOptions() if err != nil { return err @@ -224,7 +221,6 @@ func (f *FrankenPHPApp) registerModule(repl *caddy.Replacer, module *FrankenPHPM for _, w := range module.Workers { w.FileName = repl.ReplaceKnown(w.FileName, "") - w.Name = f.createUniqueWorkerName(w, serverName) workerOptions, err := w.toWorkerOptions() if err != nil { return err @@ -236,37 +232,6 @@ func (f *FrankenPHPApp) registerModule(repl *caddy.Replacer, module *FrankenPHPM return nil } -// avoid name collisions for workers -// on collision, a name is first qualified with the server name -// (":") before falling back to a numeric postfix -func (f *FrankenPHPApp) createUniqueWorkerName(wc workerConfig, serverName string) string { - if f.usedWorkerNames == nil { - f.usedWorkerNames = make(map[string]bool) - } - - if wc.Name == "" { - wc.Name, _ = fastabs.FastAbs(wc.FileName) - } - - name := wc.Name - suffix := 0 - for { - if _, ok := f.usedWorkerNames[name]; !ok { - f.usedWorkerNames[name] = true - break - } - if serverName != "" { - name = serverName + ":" + wc.Name - serverName = "" - continue - } - suffix++ - name = fmt.Sprintf("%s_%d", wc.Name, suffix) - } - - return name -} - // UnmarshalCaddyfile implements caddyfile.Unmarshaler. func (f *FrankenPHPApp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { for d.Next() { @@ -381,10 +346,13 @@ func (f *FrankenPHPApp) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if len(wc.MatchPath) != 0 { return d.Errf(`"match" can only be used in a php_server worker block, not in a global one: %q`, wc.FileName) } - // check for duplicate workers - for _, existingWorker := range f.Workers { - if existingWorker.FileName == wc.FileName { - return d.Errf("global workers must not have duplicate filenames: %q", wc.FileName) + // check for duplicate workers; background workers are keyed + // by name, several may share a script + if !wc.Background { + for _, existingWorker := range f.Workers { + if !existingWorker.Background && existingWorker.FileName == wc.FileName { + return d.Errf("global workers must not have duplicate filenames: %q", wc.FileName) + } } } diff --git a/caddy/caddy_test.go b/caddy/caddy_test.go index b7d6c231eb..2fd8b50970 100644 --- a/caddy/caddy_test.go +++ b/caddy/caddy_test.go @@ -757,6 +757,45 @@ func TestMetrics(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(ctx.GetMetricsRegistry(), strings.NewReader(expectedMetrics), "frankenphp_total_threads", "frankenphp_busy_threads")) } +// TestBackgroundWorkerFromCaddyfile starts a background worker from a +// Caddyfile and checks it runs: the sentinel its script touches appears +func TestBackgroundWorkerFromCaddyfile(t *testing.T) { + sentinel := filepath.ToSlash(filepath.Join(t.TempDir(), "bg.sentinel")) + tester := caddytest.NewTester(t) + initServer(t, tester, ` + { + skip_install_trust + admin localhost:2999 + http_port `+testPort+` + https_port 9443 + + frankenphp { + worker { + file ../testdata/bgworker/basic.php + num 1 + name bg-caddy + background + env BG_SENTINEL `+sentinel+` + } + } + } + + localhost:`+testPort+` { + route { + php { + root ../testdata + } + } + } + `, "caddyfile") + + require.Eventually(t, func() bool { + _, err := os.Stat(sentinel) + + return err == nil + }, 5*time.Second, 25*time.Millisecond, "the background worker declared in the Caddyfile did not run") +} + func TestWorkerMetrics(t *testing.T) { var wg sync.WaitGroup tester := caddytest.NewTester(t) @@ -827,7 +866,7 @@ func TestWorkerMetrics(t *testing.T) { # TYPE frankenphp_busy_threads gauge frankenphp_busy_threads 2 - # HELP frankenphp_busy_workers Number of busy PHP workers for this worker + # HELP frankenphp_busy_workers Number of busy PHP workers for this worker: processing a request, or a task for a background worker # TYPE frankenphp_busy_workers gauge frankenphp_busy_workers{worker="` + workerName + `"} 0 @@ -839,7 +878,7 @@ func TestWorkerMetrics(t *testing.T) { # TYPE frankenphp_worker_request_count counter frankenphp_worker_request_count{worker="` + workerName + `"} 10 - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, frankenphp_worker_tick for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="` + workerName + `"} 2 ` @@ -984,7 +1023,7 @@ func TestNamedWorkerMetrics(t *testing.T) { # TYPE frankenphp_busy_threads gauge frankenphp_busy_threads 2 - # HELP frankenphp_busy_workers Number of busy PHP workers for this worker + # HELP frankenphp_busy_workers Number of busy PHP workers for this worker: processing a request, or a task for a background worker # TYPE frankenphp_busy_workers gauge frankenphp_busy_workers{worker="my_app"} 0 @@ -996,7 +1035,7 @@ func TestNamedWorkerMetrics(t *testing.T) { # TYPE frankenphp_worker_request_count counter frankenphp_worker_request_count{worker="my_app"} 10 - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, frankenphp_worker_tick for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="my_app"} 2 ` @@ -1080,7 +1119,7 @@ func TestAutoWorkerConfig(t *testing.T) { # TYPE frankenphp_busy_threads gauge frankenphp_busy_threads ` + workers + ` - # HELP frankenphp_busy_workers Number of busy PHP workers for this worker + # HELP frankenphp_busy_workers Number of busy PHP workers for this worker: processing a request, or a task for a background worker # TYPE frankenphp_busy_workers gauge frankenphp_busy_workers{worker="` + workerName + `"} 0 @@ -1092,7 +1131,7 @@ func TestAutoWorkerConfig(t *testing.T) { # TYPE frankenphp_worker_request_count counter frankenphp_worker_request_count{worker="` + workerName + `"} 10 - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, frankenphp_worker_tick for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="` + workerName + `"} ` + workers + ` ` @@ -1347,6 +1386,7 @@ func TestMaxWaitTimeWorker(t *testing.T) { require.NoError(t, err) expectedMetrics := ` + # HELP frankenphp_worker_queue_depth Number of queued requests for this worker, or of tasks waiting for a thread of a background worker # TYPE frankenphp_worker_queue_depth gauge frankenphp_worker_queue_depth{worker="service"} 0 ` @@ -1447,7 +1487,7 @@ func TestMultiWorkersMetrics(t *testing.T) { # TYPE frankenphp_busy_threads gauge frankenphp_busy_threads 5 - # HELP frankenphp_busy_workers Number of busy PHP workers for this worker + # HELP frankenphp_busy_workers Number of busy PHP workers for this worker: processing a request, or a task for a background worker # TYPE frankenphp_busy_workers gauge frankenphp_busy_workers{worker="service1"} 0 @@ -1460,7 +1500,7 @@ func TestMultiWorkersMetrics(t *testing.T) { # TYPE frankenphp_worker_request_count counter frankenphp_worker_request_count{worker="service1"} 10 - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, frankenphp_worker_tick for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="service1"} 2 frankenphp_ready_workers{worker="service2"} 3 @@ -1614,7 +1654,7 @@ func TestWorkerRestart(t *testing.T) { // Check metrics expectedMetrics := ` - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, frankenphp_worker_tick for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="service"} 1 # HELP frankenphp_total_workers Total number of PHP workers for this worker @@ -1642,7 +1682,7 @@ func TestWorkerRestart(t *testing.T) { // frankenphp_ready_workers should be back to 1 even after worker restarts expectedMetrics = ` - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, frankenphp_worker_tick for background workers # TYPE frankenphp_ready_workers gauge frankenphp_ready_workers{worker="service"} 1 # HELP frankenphp_total_workers Total number of PHP workers for this worker @@ -2113,7 +2153,7 @@ func TestSymlinkWorkerBehavior(t *testing.T) { // Accessing the worker script without worker configuration MUST fail // The script checks $_SERVER['FRANKENPHP_WORKER'] and dies if not set - tester.AssertGetResponse("http://localhost:"+testPort+"/index.php", http.StatusOK, "Error: This script must be run in worker mode (FRANKENPHP_WORKER not set to '1')\n") + tester.AssertGetResponse("http://localhost:"+testPort+"/index.php", http.StatusOK, "Error: This script must be run in worker mode (FRANKENPHP_WORKER not set)\n") }) t.Run("MultipleRequests", func(t *testing.T) { diff --git a/caddy/config_test.go b/caddy/config_test.go index 607051cbd8..d71b307bb4 100644 --- a/caddy/config_test.go +++ b/caddy/config_test.go @@ -1,7 +1,6 @@ package caddy import ( - "path/filepath" "testing" "time" @@ -81,6 +80,45 @@ func TestModuleWorkerDuplicateFilenamesFail(t *testing.T) { require.Contains(t, err.Error(), "must not have duplicate filenames", "Error message should mention duplicate filenames") } +// two global background workers may share a script, like their php_server +// counterparts and the Go API: they are keyed by name +func TestGlobalBackgroundWorkersShareAFilename(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + frankenphp { + worker { + name first + file ../testdata/worker-with-env.php + num 1 + background + } + worker { + name second + file ../testdata/worker-with-env.php + num 1 + background + } + } + }`) + app := &FrankenPHPApp{} + + require.NoError(t, app.UnmarshalCaddyfile(d)) + require.Len(t, app.Workers, 2) +} + +func TestGlobalWorkerDuplicateFilenamesFail(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + frankenphp { + worker ../testdata/worker-with-env.php + worker ../testdata/worker-with-env.php + } + }`) + app := &FrankenPHPApp{} + + require.ErrorContains(t, app.UnmarshalCaddyfile(d), "must not have duplicate filenames") +} + func TestModuleWorkersWithDifferentFilenames(t *testing.T) { // Create a test configuration with different worker filenames configWithDifferentFilenames := ` @@ -249,38 +287,73 @@ func TestModuleWorkerWithCustomName(t *testing.T) { require.Equal(t, "../testdata/worker-with-env.php", module.Workers[0].FileName, "Worker should have the correct filename") } -func TestCreateUniqueWorkerNames(t *testing.T) { - app := &FrankenPHPApp{} - filename := "../testdata/worker-with-env.php" - absFileName, _ := filepath.Abs(filename) - names := make([]string, 6) - for i := range 3 { - names[i] = app.createUniqueWorkerName(workerConfig{ - FileName: filename, - Name: "custom-worker-name", - }, "") - names[i+3] = app.createUniqueWorkerName(workerConfig{ - FileName: filename, - }, "") - } - - require.Equal(t, "custom-worker-name", names[0]) - require.Equal(t, "custom-worker-name_1", names[1]) - require.Equal(t, "custom-worker-name_2", names[2]) - require.Equal(t, absFileName, names[3]) - require.Equal(t, absFileName+"_1", names[4]) - require.Equal(t, absFileName+"_2", names[5]) +func TestWorkerBackgroundConfig(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + name jobs + file ../testdata/worker-with-env.php + num 2 + background + } + } + }`) + module := &FrankenPHPModule{} + + require.NoError(t, module.UnmarshalCaddyfile(d)) + require.Len(t, module.Workers, 1) + require.True(t, module.Workers[0].Background) + require.Equal(t, "jobs", module.Workers[0].Name) } -func TestCreateUniqueWorkerNamesQualifiedByServer(t *testing.T) { - app := &FrankenPHPApp{} - wc := workerConfig{FileName: "../testdata/worker-with-env.php", Name: "queue"} - - require.Equal(t, "queue", app.createUniqueWorkerName(wc, "one.example.com")) - // on collision, the name is qualified with the server name - require.Equal(t, "two.example.com:queue", app.createUniqueWorkerName(wc, "two.example.com")) - // when the qualified name is also taken, fall back to the numeric postfix - require.Equal(t, "queue_1", app.createUniqueWorkerName(wc, "two.example.com")) - // workers without a server keep the numeric postfix behavior - require.Equal(t, "queue_2", app.createUniqueWorkerName(wc, "")) +func TestWorkerBackgroundRequiresName(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + file ../testdata/worker-with-env.php + background + } + } + }`) + module := &FrankenPHPModule{} + + err := module.UnmarshalCaddyfile(d) + require.ErrorContains(t, err, `background workers must have an explicit "name"`) +} + +func TestWorkerBackgroundRequiresNum(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + name jobs + file ../testdata/worker-with-env.php + background + } + } + }`) + module := &FrankenPHPModule{} + + err := module.UnmarshalCaddyfile(d) + require.ErrorContains(t, err, `background workers must declare "num" >= 1`) +} + +func TestWorkerBackgroundRejectsMatch(t *testing.T) { + d := caddyfile.NewTestDispenser(` + { + php_server { + worker { + name jobs + file ../testdata/worker-with-env.php + match /jobs/* + background + } + } + }`) + module := &FrankenPHPModule{} + + err := module.UnmarshalCaddyfile(d) + require.ErrorContains(t, err, `"match" is not supported for background workers`) } diff --git a/caddy/module.go b/caddy/module.go index 20dcec9ee2..ff6e2db5ff 100644 --- a/caddy/module.go +++ b/caddy/module.go @@ -315,6 +315,10 @@ func (f *FrankenPHPModule) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // Check if a worker with this filename already exists in this module fileNames := make(map[string]struct{}, len(f.Workers)) for _, w := range f.Workers { + // background workers are keyed by name, several may share a script + if w.Background { + continue + } if _, ok := fileNames[w.FileName]; ok { return fmt.Errorf(`workers in a single "php" or "php_server" block must not have duplicate filenames: %q`, w.FileName) } diff --git a/caddy/workerconfig.go b/caddy/workerconfig.go index b39eb731e0..7f5c15b253 100644 --- a/caddy/workerconfig.go +++ b/caddy/workerconfig.go @@ -22,7 +22,7 @@ import ( type workerConfig struct { mercureContext - // Name for the worker. Default: the absolute path of the worker file, postfixed with a number if the name is already used. + // Name for the worker, unique within its php_server (or among global workers). Default: the absolute path of the worker file. Name string `json:"name,omitempty"` // FileName sets the path to the worker script. FileName string `json:"file_name,omitempty"` @@ -38,6 +38,8 @@ type workerConfig struct { MatchPath []string `json:"match_path,omitempty"` // MaxConsecutiveFailures sets the maximum number of consecutive failures before panicking (defaults to 6, set to -1 to never panick) MaxConsecutiveFailures int `json:"max_consecutive_failures,omitempty"` + // Background marks this worker as a background (non-HTTP) worker. + Background bool `json:"background,omitempty"` options []frankenphp.WorkerOption } @@ -139,8 +141,10 @@ func unmarshalWorker(d *caddyfile.Dispenser) (workerConfig, error) { } wc.MaxConsecutiveFailures = v + case "background": + wc.Background = true default: - return wc, wrongSubDirectiveError("worker", "name, file, num, env, watch, match, max_consecutive_failures, max_threads", v) + return wc, wrongSubDirectiveError("worker", "name, file, num, env, watch, match, max_consecutive_failures, max_threads, background", v) } } @@ -148,6 +152,18 @@ func unmarshalWorker(d *caddyfile.Dispenser) (workerConfig, error) { return wc, d.Err(`the "file" argument must be specified`) } + if wc.Background { + if wc.Name == "" { + return wc, d.Err(`background workers must have an explicit "name"`) + } + if len(wc.MatchPath) != 0 { + return wc, d.Err(`"match" is not supported for background workers`) + } + if wc.Num < 1 { + return wc, d.Err(`background workers must declare "num" >= 1`) + } + } + if frankenphp.EmbeddedAppPath != "" && filepath.IsLocal(wc.FileName) { wc.FileName = filepath.Join(frankenphp.EmbeddedAppPath, wc.FileName) } @@ -166,6 +182,10 @@ func (wc *workerConfig) toWorkerOptions() ([]frankenphp.WorkerOption, error) { // options collected while provisioning the module, e.g. the Mercure hub opts = append(opts, wc.options...) + if wc.Background { + opts = append(opts, frankenphp.WithWorkerBackground()) + } + // copy the caddy match logic and create a unique matcher function for this worker // inject the matcher into frankenphp if len(wc.MatchPath) > 0 { diff --git a/cgi.go b/cgi.go index 85b6a4469a..25a7b87e44 100644 --- a/cgi.go +++ b/cgi.go @@ -225,7 +225,7 @@ func splitCgiPath(fc *frankenPHPContext) { // see if a php_server worker or global worker matches the request path // aka: root + request path == worker.filename if fc.worker = fc.server.workersByPath[fc.scriptFilename]; fc.worker == nil { - fc.worker = globalWorkersByPath[fc.scriptFilename] + fc.worker = fallbackServer.workersByPath[fc.scriptFilename] } } diff --git a/context.go b/context.go index d184965fea..b0fbcd52a4 100644 --- a/context.go +++ b/context.go @@ -130,20 +130,14 @@ func newWorkerDummyContext(w *worker) (*frankenPHPContext, error) { return nil, err } - server := w.server - if server == nil { - // global worker, not associated with a server - server = fallbackServer - } - fc := &frankenPHPContext{ done: make(chan any), ctx: r.Context(), - server: server, + server: w.server, request: r, startedAt: time.Now(), // startup output of a scoped worker belongs to its server's logger - logger: server.logger, + logger: w.server.logger, worker: w, } @@ -160,11 +154,6 @@ func newWorkerDummyContext(w *worker) (*frankenPHPContext, error) { // newContextFromMessage creates a context from a message (external workers) func newContextFromMessage(message any, rw http.ResponseWriter, ctx context.Context, w *worker) *frankenPHPContext { - server := w.server - if server == nil { - server = fallbackServer - } - if ctx == nil { ctx = globalCtx } @@ -172,9 +161,9 @@ func newContextFromMessage(message any, rw http.ResponseWriter, ctx context.Cont return &frankenPHPContext{ done: make(chan any), startedAt: time.Now(), - server: server, + server: w.server, worker: w, - logger: server.logger, + logger: w.server.logger, responseWriter: rw, handlerParameters: message, ctx: ctx, diff --git a/docs/config.md b/docs/config.md index 281f05dc75..5c055400b0 100644 --- a/docs/config.md +++ b/docs/config.md @@ -109,8 +109,9 @@ You can also explicitly configure FrankenPHP using the [global option](https://c num # Sets the number of PHP threads to start, defaults to 2x the number of available CPUs. env # Sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables. watch # Sets the path to watch for file changes. Can be specified more than once for multiple paths. - name # Sets the name of the worker, used in logs and metrics. Default: absolute path of worker file + name # Sets the name of the worker, used in logs and metrics. Must be unique among global workers. Default: absolute path of the worker file. max_consecutive_failures # Sets the maximum number of consecutive failures before the worker is considered unhealthy, -1 means the worker will always restart. Default: 6. + background # EXPERIMENTAL: marks this worker as a background (non-HTTP) worker; it runs the script in a loop without serving requests, "name" is required, "match" is not allowed. The script is ready once it calls frankenphp_worker_tick(); an exit before that counts as a failure, and max_execution_time applies until that call, not after. $_SERVER['FRANKENPHP_WORKER_BACKGROUND'] holds its name, and FRANKENPHP_WORKER is not set in it. Its threads come on top of num_threads and max_threads, and max_threads is not allowed on it. } } } @@ -187,17 +188,18 @@ php_server [] { root # Sets the root folder to the site. Default: `root` directive. split_path # Sets the substrings for splitting the URI into two parts. The first matching substring will be used to split the "path info" from the path. The first piece is suffixed with the matching substring and will be assumed as the actual resource (CGI script) name. The second piece will be set to PATH_INFO for the script to use. Default: `.php` resolve_root_symlink false # Disables resolving the `root` directory to its actual value by evaluating a symbolic link, if one exists (enabled by default). - name # Sets the name for this server, used to attribute workers, metrics and logs. Default: the first host matcher of the enclosing route, or the first listener address. + name # Sets the name for this server, used to attribute workers, metrics and logs. Default: the first host matcher of the enclosing route, or the first listener address. Suffixed with a number if another php_server resolves to the same name. env # Sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables. file_server off # Disables the built-in file_server directive. request_body_timeout # Sets an idle timeout on request body reads: a stalled (slow POST) client is cut off while a steady upload of any size succeeds. Default: 60s. Set to 0 to disable. worker { # Creates a worker specific to this server. Can be specified more than once for multiple workers. file # Sets the path to the worker script, can be relative to the php_server root num # Sets the number of PHP threads to start, defaults to 2x the number of available - name # Sets the name for the worker, used in logs and metrics. Default: absolute path of worker file. Postfixed with a number if name is already in use. + name # Sets the name for the worker, used in logs and metrics. Must be unique within this php_server. In logs and metrics, the worker is reported as ":". Default: absolute path of the worker file. watch # Sets the path to watch for file changes. Can be specified more than once for multiple paths. env # Sets an extra environment variable to the given value. Can be specified more than once for multiple environment variables. Environment variables for this worker are also inherited from the php_server parent, but can be overwritten here. match # match the worker to a path pattern. Overrides try_files and can only be used in the php_server directive. + background # EXPERIMENTAL: marks this worker as a background (non-HTTP) worker; it runs the script in a loop without serving requests, "name" is required, "match" is not allowed. The script is ready once it calls frankenphp_worker_tick(); an exit before that counts as a failure, and max_execution_time applies until that call, not after. $_SERVER['FRANKENPHP_WORKER_BACKGROUND'] holds its name, and FRANKENPHP_WORKER is not set in it. Its threads come on top of num_threads and max_threads, and max_threads is not allowed on it. } worker # Can also use the short form like in the global frankenphp block. } diff --git a/docs/library.md b/docs/library.md index 7782f03182..03f0ef5898 100644 --- a/docs/library.md +++ b/docs/library.md @@ -60,6 +60,10 @@ err := frankenphp.Init( Workers declared without a server scope are global: they match by file path on any server. Since a global worker has no set of requests to match against, combining `WithWorkerMatcher()` with a global worker is a configuration error and `Init()` rejects it. +Worker names are unique within their server (or among global workers), so two servers may each declare a worker named `queue`. The script sees the declared name, while metrics and logs report a server-scoped worker as `:`; server names are made unique with a numeric suffix when needed. `WithWorkerName()` resolves a name within the request's server first, then among global workers. + +`WithWorkerBackground()` declares a [background worker](worker.md#background-workers), which runs outside the request cycle. + ## Per-request options `Server.ServeHTTP()` accepts `RequestOption`s to override the server configuration for a single request, e.g. `WithRequestDocumentRoot()`, `WithRequestSplitPath()`, `WithRequestEnv()` or `WithRequestLogger()`. diff --git a/docs/metrics.md b/docs/metrics.md index 932707265a..9456842efc 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -16,15 +16,17 @@ When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenP - `frankenphp_busy_threads`: The number of PHP threads currently processing a request (running workers always consume a thread). - `frankenphp_queue_depth`: The number of regular queued requests. - `frankenphp_total_workers{worker="[worker_name]"}`: The total number of workers. -- `frankenphp_busy_workers{worker="[worker_name]"}`: The number of workers currently processing a request. +- `frankenphp_busy_workers{worker="[worker_name]"}`: The number of workers currently processing a request, or a task for a background worker. - `frankenphp_worker_request_time{worker="[worker_name]"}`: The time spent processing requests by all workers. - `frankenphp_worker_request_count{worker="[worker_name]"}`: The number of requests processed by all workers. -- `frankenphp_ready_workers{worker="[worker_name]"}`: The number of workers that have called `frankenphp_handle_request` at least once. +- `frankenphp_ready_workers{worker="[worker_name]"}`: The number of workers that have reached their ready point at least once: `frankenphp_handle_request()` for HTTP workers, `frankenphp_worker_tick()` for background workers. - `frankenphp_worker_crashes{worker="[worker_name]"}`: The number of times a worker has unexpectedly terminated. - `frankenphp_worker_restarts{worker="[worker_name]"}`: The number of times a worker has been deliberately restarted. -- `frankenphp_worker_queue_depth{worker="[worker_name]"}`: The number of queued requests. +- `frankenphp_worker_queue_depth{worker="[worker_name]"}`: The number of queued requests, or of tasks waiting for a thread of a background worker. +- `frankenphp_worker_task_count{worker="[worker_name]",outcome="[outcome]"}`: The number of tasks sent to a background worker, by outcome: `completed`, `aborted` (the script ended with the task open), `abandoned` (the sender closed its stream first) or `timeout` (no thread picked the task up in time). +- `frankenphp_worker_task_time{worker="[worker_name]"}`: The time spent on tasks by all threads of a background worker, from pickup to the close of the task's stream. -For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used. +`[worker_name]` is the worker name from the Caddyfile, or the absolute path of the worker file when it has none. Workers of a `php_server` block are prefixed with the name of that block: `:`. They used to be reported under their bare name unless two blocks declared the same one, so dashboards and alerts built on those series need the prefix. ## Threads State Endpoint @@ -73,8 +75,8 @@ Each entry in `ThreadDebugStates` contains: | `Index` | integer | The index of the thread. | | `Name` | string | The name of the thread (e.g., the worker file path). | | `State` | string | The internal state of the thread (e.g., `ready`, `shutting down`). | -| `IsWaiting` | boolean | Whether the thread is waiting for a request. | -| `IsBusy` | boolean | Whether the thread is currently processing a request. | +| `IsWaiting` | boolean | Whether the thread is waiting for a request, or for a task in a background worker. | +| `IsBusy` | boolean | Whether the thread is currently processing a request, or a task in a background worker. | | `WaitingSinceMilliseconds` | integer | How long the thread has been idle, in milliseconds. `0` if the thread is busy. | | `CurrentURI` | string | The URI currently being processed. Empty if the thread is idle. | | `CurrentMethod` | string | The HTTP method of the current request (e.g., `GET`, `POST`). Empty if the thread is idle. | diff --git a/docs/worker.md b/docs/worker.md index 466c7cf684..ce31e8a61f 100644 --- a/docs/worker.md +++ b/docs/worker.md @@ -199,6 +199,113 @@ frankenphp { } ``` +## Background workers + +This feature is experimental. + +A background worker runs its script in a loop outside the HTTP request cycle, on its own PHP thread. It is declared like any worker, with the `background` option; `name` is required and `num` must be at least 1: + +```caddyfile +php_server { + worker { + file jobs.php + num 1 + name jobs + background + } +} +``` + +The script calls `frankenphp_worker_tick()` once it is set up. The first call marks the worker ready: the server start waits for that point, and an exit before it counts as a failure. Until that call, `max_execution_time` applies as in any request, 30 seconds by default: a setup that outlives it is ended and counts as a boot failure, so raise the limit in `php_ini`, or call `set_time_limit()` in the script, when the setup legitimately takes longer. From the first call on, the run has no time limit, like the CLI. Every call returns `false` once FrankenPHP drains the worker, on shutdown, reboot or restart, and `true` otherwise. It never blocks and never hands out work: like `frankenphp_handle_request()`, it is where the runtime and the script meet. Between two calls, the script waits on the stream returned by `frankenphp_get_worker_handle()`, alone or together with its own streams. The stream becomes readable when FrankenPHP needs the script's attention, and `frankenphp_worker_tick()` consumes whatever was written on it, so the script does not read the stream itself. It is readable once right after the script starts, so a loop that services it ticks by itself and the worker is ready as soon as its loop runs. + +```php +stop(); + } +}); + +// the script's own watchers go here + +EventLoop::run(); +``` + +The wake-up sent at start makes the callback run as soon as the loop does, which is when the worker becomes ready. The polling API of PHP 8.6 works the same way: wrap the stream in a `StreamPollHandle`, add it to a context, and call `frankenphp_worker_tick()` when it triggers. + +`$_SERVER['FRANKENPHP_WORKER_BACKGROUND']` holds the declared name. `FRANKENPHP_WORKER`, the variable of HTTP workers, is not set, so a script serving both roles tests which of the two is set. Background threads come on top of `num_threads` and `max_threads`; they don't autoscale, so `max_threads` is not allowed on them. From Go, declare one with `WithWorkerBackground()`. + +### Sharing state with background workers + +A background worker publishes a snapshot with `frankenphp_set_vars()`; requests and other workers read it with `frankenphp_get_vars()`, by worker name, resolved like requests are: within the `php_server`, then among global workers. Values must be null, scalars, arrays or enums. Each call replaces the whole snapshot and readers get a copy, so the worker can publish at any time and a request always sees a consistent one. Publish before the first `frankenphp_worker_tick()` and the snapshot is in place before the server accepts requests; while the worker restarts, readers keep getting the last one. + +```php +// background worker +frankenphp_set_vars(['maintenance' => false, 'flags' => ['beta' => true]]); +$handle = frankenphp_get_worker_handle(); +// ... + +// request, HTTP worker or another background worker +$vars = frankenphp_get_vars('config'); +``` + +`frankenphp_get_vars()` blocks until the worker reached its ready point, which can only happen between background workers reading each other while booting; a cycle between them throws instead of hanging. It also throws when the name is unknown, or when the worker is ready but has not published anything. + +### Sending tasks to background workers + +A request, an HTTP worker or another background worker hands work to a background worker with `frankenphp_send_task()`, by worker name, resolved like `frankenphp_get_vars()` does. The payload follows the same rules as `frankenphp_set_vars()`: null, scalars, arrays or enums. The call blocks until a thread of the worker picks the task up and throws if none did before the timeout, so a busy worker pushes back on its senders instead of queueing without bounds. It returns a stream: `frankenphp_read_task()` blocks for the next update and returns `null` once the worker completed the task, and `stream_select()` works on the stream to wait on several tasks or to bound the wait. Closing the stream abandons the task. + +On the worker side, each task sent wakes one parked thread of the worker through its handle, in the loop of the previous section: `frankenphp_worker_tick()` consumes the wake-up, which is not a count and in a pool may belong to a task a sibling thread took. `frankenphp_receive_task()` dequeues a task without blocking, `[$stream, $payload]`, or `null` when another thread of the pool got there first, so the example below drains the queue on each wake-up and treats `null` as the normal outcome. A thread that ticks while tasks are queued finds its handle readable at once, whichever loop shape it uses. `frankenphp_update_task()` sends progress or a result back and closing the stream completes the task; a script that ends with the stream still open, a close from a destructor or a shutdown function at that point included, makes the sender's next `frankenphp_read_task()` throw. When the sender closes its stream instead, the worker's stream reaches EOF, so `stream_select()` or `feof()` on it tell a long task that nobody waits for its result, and `frankenphp_update_task()` throws. + +```php +// background worker +$handle = frankenphp_get_worker_handle(); + +while (frankenphp_worker_tick()) { + $read = [$handle]; + $write = $except = null; + stream_select($read, $write, $except, null); + + while ($task = frankenphp_receive_task()) { + [$stream, $payload] = $task; + frankenphp_update_task($stream, ['progress' => 50]); + frankenphp_update_task($stream, ['result' => process($payload)]); + fclose($stream); + } +} + +// request, HTTP worker or another background worker +$task = frankenphp_send_task('jobs', ['file' => 'photo.jpg']); +while (null !== $update = frankenphp_read_task($task)) { + // ['progress' => 50], then ['result' => ...] +} +``` + +Sixteen updates are buffered per task; past that, `frankenphp_update_task()` waits for the sender to read, and it throws once the sender closed its stream. The streams of a task are backed by eventfd descriptors on Linux, pooled between tasks, and by a socket pair elsewhere. + ## Superglobals behavior [PHP superglobals](https://www.php.net/manual/language.variables.superglobals.php) (`$_SERVER`, `$_ENV`, `$_GET`...) diff --git a/frankenphp.c b/frankenphp.c index 2378ac8ff6..4d3e7fdfd1 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -13,11 +13,14 @@ #include #ifdef PHP_WIN32 #include +#include #else #include #endif +#include
#include #include +#include #include #include #include @@ -28,6 +31,7 @@ #include #include #ifndef PHP_WIN32 +#include #include #endif #if defined(__linux__) @@ -52,12 +56,7 @@ #include "_cgo_export.h" #include "frankenphp_arginfo.h" -#ifdef FRANKENPHP_TEST -/* The persistent_zval helpers are only compiled in when a consumer needs - * them. The step that lands the first real caller (background workers) - * will drop this guard. */ #include "zval.h" -#endif #if defined(PHP_WIN32) && defined(ZTS) ZEND_TSRMLS_CACHE_DEFINE() @@ -126,6 +125,18 @@ HashTable *main_thread_env = NULL; static THREAD_LOCAL uintptr_t thread_index; static THREAD_LOCAL bool is_worker_thread = false; +static THREAD_LOCAL bool is_background_worker = false; +/* Stop socket pair of a background worker thread: [0] is the script's end, + * exposed via frankenphp_get_worker_handle(); [1] is transferred to the Go + * side, which closes it to signal a drain. */ +static THREAD_LOCAL php_socket_t worker_stop_socks[2] = {SOCK_ERR, SOCK_ERR}; +/* set by the first frankenphp_worker_tick() of the current run, the ready + * point of a background worker */ +static THREAD_LOCAL bool worker_ticked = false; +/* the stream of the current run, see frankenphp_get_worker_handle(); the + * cache holds a ref, and the resource list of the run frees it at request + * shutdown, so the pointer is only reset, never released, between runs */ +static THREAD_LOCAL zend_resource *worker_handle_res = NULL; static THREAD_LOCAL HashTable *sandboxed_env = NULL; /* prepared_env holds entries from php(_server)'s `env KEY VAL`, exposed to * getenv() and merged into $_ENV when 'E' is in variables_order. Separate from @@ -342,7 +353,261 @@ void frankenphp_release_thread_for_kill(force_kill_slot slot) { #endif } +/* Socket pairs of background workers: the handle of a thread, see + * frankenphp_get_worker_handle(), and one per task, see + * frankenphp_send_task(). One end is exposed to a PHP script as a stream, + * the other is held by the Go side, which writes wake-ups to it and closes + * it to land EOF on the script's end, so a stream_select() or a blocking + * read there returns. A socket pair rather than a pipe because on Windows + * PHP's php_select() only really waits on sockets: before 8.5 it reports + * any other handle as always ready. */ +static void frankenphp_sock_close(php_socket_t s) { + if (s == SOCK_ERR) { + return; + } + /* php_network.h maps closesocket to close outside Windows */ + closesocket(s); +} + +/* keep the pair out of processes the script may spawn: a child holding the + * Go side's end would keep the script's end from ever reaching EOF */ +static void frankenphp_sock_no_inherit(php_socket_t s) { +#ifdef PHP_WIN32 + SetHandleInformation((HANDLE)s, HANDLE_FLAG_INHERIT, 0); +#else + fcntl(s, F_SETFD, FD_CLOEXEC); +#endif +} + +/* Opens a pair: [0] for the script, [1] for the Go side. The Go side's end + * never blocks: its writes are wake-ups, a full buffer means the peer has + * plenty of unread ones already. */ +static int frankenphp_sock_pair_open(php_socket_t socks[2]) { +#ifdef PHP_WIN32 + /* PHP's emulation, a loopback TCP pair; it only accepts AF_INET, listens + * on INADDR_ANY and accepts the first peer, so check the pair is ours */ + if (socketpair(AF_INET, SOCK_STREAM, 0, socks) != 0) { + socks[0] = SOCK_ERR; + socks[1] = SOCK_ERR; + + return -1; + } + + struct sockaddr_in peer = {0}, local = {0}; + int peer_len = sizeof(peer), local_len = sizeof(local); + if (getpeername(socks[0], (struct sockaddr *)&peer, &peer_len) != 0 || + getsockname(socks[1], (struct sockaddr *)&local, &local_len) != 0 || + peer.sin_port != local.sin_port || + peer.sin_addr.s_addr != local.sin_addr.s_addr) { + frankenphp_sock_close(socks[0]); + frankenphp_sock_close(socks[1]); + socks[0] = SOCK_ERR; + socks[1] = SOCK_ERR; + + return -1; + } +#else +#ifdef SOCK_CLOEXEC + int type = SOCK_STREAM | SOCK_CLOEXEC; +#else + int type = SOCK_STREAM; +#endif + if (socketpair(AF_UNIX, type, 0, socks) != 0) { + socks[0] = SOCK_ERR; + socks[1] = SOCK_ERR; + + return -1; + } +#endif + /* redundant where SOCK_CLOEXEC applied; a fork()ed child (pcntl) still + * inherits both ends and delays the EOF until it exits */ + frankenphp_sock_no_inherit(socks[0]); + frankenphp_sock_no_inherit(socks[1]); + php_set_sock_blocking(socks[1], 0); +#ifdef PHP_WIN32 + /* loopback TCP: a byte-sized wake-up must not wait for Nagle and the + * delayed ACK */ + int nodelay = 1; + setsockopt(socks[0], IPPROTO_TCP, TCP_NODELAY, (const char *)&nodelay, + sizeof(nodelay)); + setsockopt(socks[1], IPPROTO_TCP, TCP_NODELAY, (const char *)&nodelay, + sizeof(nodelay)); +#endif +#ifdef SO_NOSIGPIPE + /* a wake-up to a closed peer must fail, not raise (MSG_NOSIGNAL elsewhere) */ + int one = 1; + setsockopt(socks[1], SOL_SOCKET, SO_NOSIGPIPE, &one, sizeof(one)); +#endif + + return 0; +} + +/* best-effort wake-up on the Go side's end of a pair */ +static void frankenphp_sock_send(php_socket_t s, const char *buf, size_t len) { +#ifdef MSG_NOSIGNAL + int flags = MSG_NOSIGNAL; +#else + int flags = 0; +#endif + (void)send(s, buf, (int)len, flags); +} + +static void frankenphp_worker_close_stop_socks(void) { + for (int i = 0; i < 2; i++) { + frankenphp_sock_close(worker_stop_socks[i]); + worker_stop_socks[i] = SOCK_ERR; + } +} + +/* Marks the calling thread as a background worker, opens its stop socket + * pair and transfers the Go side's end to the caller (clearing the TLS slot + * so a later recycle won't double-close it). Returns -1 if the pair could + * not be created. max_execution_time is disarmed after php_request_startup() + * re-arms it, see php_thread(). */ +intptr_t frankenphp_set_background_worker_and_get_stop_sock(void) { + is_background_worker = true; + worker_ticked = false; + worker_handle_res = NULL; + + frankenphp_worker_close_stop_socks(); + if (frankenphp_sock_pair_open(worker_stop_socks) != 0) { + return -1; + } + + /* One wake-up right away, so a script that registers its handle with an + * event loop and runs it ticks on its own: readiness then means the loop + * serviced the handle once. The first frankenphp_worker_tick() consumes + * it. Nothing to do on failure, the script then has to tick by itself. */ + const char wakeup = '\n'; +#ifdef MSG_NOSIGNAL + send(worker_stop_socks[1], &wakeup, 1, MSG_NOSIGNAL); +#else + send(worker_stop_socks[1], &wakeup, 1, 0); +#endif + + intptr_t s = (intptr_t)worker_stop_socks[1]; + worker_stop_socks[1] = SOCK_ERR; + + return s; +} + +/* Closes the Go side's end of a pair, which lands as EOF on the script's + * end so its stream_select() or blocking read returns promptly. */ +void frankenphp_close_sock(intptr_t s) { + if (s < 0) { + return; + } + frankenphp_sock_close((php_socket_t)s); +} + +/* Closes the Go side's end of a stop socket pair, which lands as EOF on the + * script's end so its wait returns promptly. Closing alone only does while + * no other process holds a copy of it, and a pcntl_fork() child inherits + * every descriptor of the process, the pairs of the other threads included: + * shutting the write direction down sends the FIN regardless. */ +void frankenphp_worker_close_stop_sock(intptr_t s) { + if (s < 0) { + return; + } +#ifdef PHP_WIN32 + shutdown((php_socket_t)s, SD_SEND); +#else + shutdown((php_socket_t)s, SHUT_WR); +#endif + frankenphp_sock_close((php_socket_t)s); +} + +/* Wakes a background worker thread with a line on its handle, one per task + * sent to the worker, see frankenphp_send_task(). The line is a wake-up + * rather than a description: it says something may be pending, the script + * finds out what by polling. Its content is therefore not part of the + * contract, see frankenphp_receive_task(). */ +void frankenphp_worker_signal_task(intptr_t s) { + frankenphp_sock_send((php_socket_t)s, "\n", 1); +} + +/* Task channels: one descriptor per side of a task, the sender's [0] and + * the receiver's [1], each waited on by its stream and signaled by the other + * side through the Go side. On Linux they are eventfds: a counter, no + * buffer, nothing to close between two tasks, so the Go side pools them. + * Elsewhere a socket pair, for Windows's php_select(); a signal to one end + * is a byte written to the other. Both descriptors are non-blocking: waits + * go through poll(), consuming a signal never blocks. Signals and events + * match one to one, EFD_SEMAPHORE makes a read consume a single one. */ +#ifdef __linux__ +#include +#define FRANKENPHP_TASK_CHAN_EVENTFD 1 +#endif + +int frankenphp_task_chan_open(intptr_t fds[2]) { +#ifdef FRANKENPHP_TASK_CHAN_EVENTFD + int a = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK | EFD_SEMAPHORE); + if (a < 0) { + return -1; + } + int b = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK | EFD_SEMAPHORE); + if (b < 0) { + close(a); + + return -1; + } + fds[0] = a; + fds[1] = b; +#else + php_socket_t pair[2]; + if (frankenphp_sock_pair_open(pair) != 0) { + return -1; + } + php_set_sock_blocking(pair[0], 0); + fds[0] = (intptr_t)pair[0]; + fds[1] = (intptr_t)pair[1]; +#endif + + return 0; +} + +/* Wakes the side waiting on fds[side]. */ +void frankenphp_task_chan_signal(intptr_t fd0, intptr_t fd1, int side) { +#ifdef FRANKENPHP_TASK_CHAN_EVENTFD + uint64_t one = 1; + (void)!write((int)(side ? fd1 : fd0), &one, sizeof(one)); +#else + /* a byte on one end lands on the other */ + frankenphp_sock_send((php_socket_t)(side ? fd0 : fd1), "1", 1); +#endif +} + +/* Consumes one signal, false when none is pending. */ +bool frankenphp_task_chan_consume(intptr_t fd) { +#ifdef FRANKENPHP_TASK_CHAN_EVENTFD + uint64_t v; + + return read((int)fd, &v, sizeof(v)) == (ssize_t)sizeof(v); +#else + char b; + + return recv((php_socket_t)fd, &b, 1, 0) == 1; +#endif +} + +/* Empties a descriptor before its pair goes back to the pool. */ +void frankenphp_task_chan_drain(intptr_t fd) { + while (frankenphp_task_chan_consume(fd)) { + } +} + void frankenphp_update_local_thread_context(bool is_worker) { + /* A thread that ran a background worker can be recycled into an HTTP + * worker or a regular request thread: reset the bg TLS so + * frankenphp_get_worker_handle() rejects callers again, and release the + * stop socket. The streams handed out by frankenphp_get_worker_handle() + * do not own it and were destroyed by request shutdown. */ + if (is_background_worker) { + is_background_worker = false; + worker_handle_res = NULL; + frankenphp_worker_close_stop_socks(); + } + is_worker_thread = is_worker; /* workers should keep running if the user aborts the connection */ @@ -857,6 +1122,15 @@ PHP_FUNCTION(frankenphp_handle_request) { RETURN_THROWS(); } + if (is_background_worker) { + /* background workers never receive HTTP requests */ + zend_throw_exception( + spl_ce_RuntimeException, + "frankenphp_handle_request() cannot be called from a background worker", + 0); + RETURN_THROWS(); + } + #ifdef ZEND_MAX_EXECUTION_TIMERS /* Disable timeouts while waiting for a request to handle */ zend_unset_timeout(); @@ -1017,6 +1291,593 @@ PHP_FUNCTION(frankenphp_log) { } } +/* Ops of the streams returned by frankenphp_get_worker_handle(): the socket + * ops, except that closing a stream leaves the socket alone: it belongs to + * the thread, every handle of a run shares it, and it is closed at the next + * run setup or on thread exit. Initialized in MINIT. */ +static php_stream_ops frankenphp_worker_handle_ops; + +static int frankenphp_worker_handle_close(php_stream *stream, + int close_handle) { + (void)close_handle; + + /* free the stream data only, never the shared socket */ + return php_stream_socket_ops.close(stream, 0); +} + +PHP_FUNCTION(frankenphp_get_worker_handle) { + ZEND_PARSE_PARAMETERS_NONE(); + + if (!is_background_worker) { + zend_throw_exception(spl_ce_RuntimeException, + "frankenphp_get_worker_handle() can only be called " + "from a background worker", + 0); + RETURN_THROWS(); + } + + if (worker_stop_socks[0] == SOCK_ERR) { + zend_throw_exception(spl_ce_RuntimeException, + "the background worker stop socket is not available", + 0); + RETURN_THROWS(); + } + + /* One stream per run: the same resource is returned until the script + * closes it, so fetching the handle in a loop does not grow the resource + * list of a run that never ends. The stream does not own the socket (see + * frankenphp_worker_handle_ops), so closing it never affects a later one + * and the EOF of a drain reaches all. */ + if (worker_handle_res != NULL) { + if (worker_handle_res->type == php_file_le_stream()) { + GC_ADDREF(worker_handle_res); + RETURN_RES(worker_handle_res); + } + /* closed by the script: drop the cache's ref */ + zend_list_delete(worker_handle_res); + worker_handle_res = NULL; + } + + php_stream *stream = + php_stream_sock_open_from_socket(worker_stop_socks[0], NULL); + if (stream == NULL) { + zend_throw_exception(spl_ce_RuntimeException, + "failed to create a stream over the stop socket", 0); + RETURN_THROWS(); + } + + /* a blocking read is a valid way to park: wait without the + * default_socket_timeout wake-ups */ + ((php_netstream_data_t *)stream->abstract)->timeout.tv_sec = -1; + + stream->ops = &frankenphp_worker_handle_ops; + + php_stream_to_zval(stream, return_value); + worker_handle_res = Z_RES_P(return_value); + GC_ADDREF(worker_handle_res); +} + +/* The ready point of a background worker and its liveness check, the + * background analog of frankenphp_handle_request(): the first call of a run + * reports the worker ready, and every call returns false once FrankenPHP + * drains it. It never blocks and never hands out work: the script waits on + * its handle, alone or with its own streams, and calls this when the handle + * is readable. Whatever the runtime writes on the handle to wake the script + * up is consumed here, so the script never has to read the handle and the + * protocol on it stays private. */ +PHP_FUNCTION(frankenphp_worker_tick) { + ZEND_PARSE_PARAMETERS_NONE(); + + if (!is_background_worker) { + zend_throw_exception(spl_ce_RuntimeException, + "frankenphp_worker_tick() can only be called from a " + "background worker", + 0); + RETURN_THROWS(); + } + + if (worker_stop_socks[0] == SOCK_ERR) { + zend_throw_exception(spl_ce_RuntimeException, + "the background worker stop socket is not available", + 0); + RETURN_THROWS(); + } + + if (!worker_ticked) { + worker_ticked = true; + /* The bootstrap ran under max_execution_time like any request; the loop + * that starts now has no time limit, like the CLI. Nothing re-arms the + * timer past this point: php_execute_script() did so before the script + * started, request shutdown comes after it ended. */ + zend_unset_timeout(); + go_frankenphp_background_worker_ready(frankenphp_thread_index()); + } + + /* consume the wake-ups without blocking; EOF is the drain */ + char buf[64]; + for (;;) { + if (php_pollfd_for_ms(worker_stop_socks[0], PHP_POLLREADABLE, 0) <= 0) { + /* nothing pending, or a transient poll error: still running */ + break; + } + +#ifdef PHP_WIN32 + int n = recv(worker_stop_socks[0], buf, (int)sizeof(buf), 0); +#else + ssize_t n = recv(worker_stop_socks[0], buf, sizeof(buf), 0); +#endif + if (n == 0) { + /* the Go side closed its end: drained */ + RETURN_FALSE; + } + if (n < 0) { + int err = php_socket_errno(); + if (err == EINTR || PHP_IS_TRANSIENT_ERROR(err)) { + break; + } + + /* a broken socket carries no drain anymore, stop the loop */ + RETURN_FALSE; + } + } + + /* the script is about to wait on the handle: park the thread, or make the + * wait return at once when tasks are queued, see + * go_frankenphp_background_worker_park */ + go_frankenphp_background_worker_park(frankenphp_thread_index()); + + RETURN_TRUE; +} + +/* Shared vars of background workers, see frankenphp_set_vars() and + * frankenphp_get_vars(): the persistent tables live in slots owned by the + * Go side, which copies and frees them through these two helpers. */ +void frankenphp_vars_to_request(zval *return_value, HashTable *table) { + zval persistent; + ZVAL_ARR(&persistent, table); + persistent_zval_to_request(return_value, &persistent); +} + +void frankenphp_vars_free(HashTable *table) { + zval persistent; + ZVAL_ARR(&persistent, table); + persistent_zval_free(&persistent); +} + +PHP_FUNCTION(frankenphp_set_vars) { + zval *vars; + ZEND_PARSE_PARAMETERS_START(1, 1) + Z_PARAM_ARRAY(vars) + ZEND_PARSE_PARAMETERS_END(); + + if (!is_background_worker) { + zend_throw_exception( + spl_ce_RuntimeException, + "frankenphp_set_vars() can only be called from a background worker", 0); + RETURN_THROWS(); + } + + /* validate the whole tree first: persist and free recurse without a + * guard of their own */ + if (!persistent_zval_validate(vars)) { + zend_value_error("frankenphp_set_vars(): values must be null, scalars, " + "arrays or enums, nested no deeper than %d levels", + PERSISTENT_ZVAL_MAX_DEPTH); + RETURN_THROWS(); + } + + zval persistent; + persistent_zval_persist(&persistent, vars); + + HashTable *old = + go_frankenphp_set_vars(frankenphp_thread_index(), Z_ARRVAL(persistent)); + if (old != NULL) { + frankenphp_vars_free(old); + } +} + +PHP_FUNCTION(frankenphp_get_vars) { + zend_string *name; + ZEND_PARSE_PARAMETERS_START(1, 1) + Z_PARAM_STR(name) + ZEND_PARSE_PARAMETERS_END(); + + char *error = go_frankenphp_get_vars( + frankenphp_thread_index(), ZSTR_VAL(name), ZSTR_LEN(name), return_value); + if (error != NULL) { + zend_throw_exception(spl_ce_RuntimeException, error, 0); + free(error); + RETURN_THROWS(); + } +} + +/* Tasks, see frankenphp_send_task(): the sender hands a persistent copy of + * the payload to the Go side, which queues it for the named background + * worker and wakes one of its threads; frankenphp_receive_task() dequeues it + * there. Updates flow back the same way, persistent copies through the Go + * side. Each side has a stream over its descriptor of the task's channel: + * the stream carries no data, it is what stream_select() waits on and what + * fclose() ends, and the Go side holds the state the functions report. The + * descriptors belong to the task until both sides closed. */ +typedef struct { + uintptr_t task; + intptr_t fd; /* the side's descriptor, see frankenphp_task_chan_open */ + int timeout_ms; /* stream_set_timeout(), -1 waits forever */ + bool sender; + bool timed_out; + bool settled; /* the end of the task was reported, its signal consumed */ +} frankenphp_task_stream_data; + +static ssize_t frankenphp_task_stream_write(php_stream *stream, const char *buf, + size_t count) { + (void)stream; + (void)buf; + (void)count; + + return -1; +} + +/* the data goes through the frankenphp_*_task() functions */ +static ssize_t frankenphp_task_stream_read(php_stream *stream, char *buf, + size_t count) { + (void)buf; + (void)count; + frankenphp_task_stream_data *data = stream->abstract; + if (go_frankenphp_task_side_gone(data->task, data->sender)) { + stream->eof = 1; + } + + return -1; +} + +/* Closing the receiver's stream completes the task, unless the close is the + * resource cleanup of request shutdown, where the script ended with the task + * open and the sender is told so; closing the sender's abandons it. The Go + * side learns it before signaling the other side, which then finds it. */ +static int frankenphp_task_stream_close(php_stream *stream, int close_handle) { + (void)close_handle; + frankenphp_task_stream_data *data = stream->abstract; + if (data->sender) { + go_frankenphp_task_sender_close(data->task); + } else { + go_frankenphp_task_receiver_close(data->task, + (EG(flags) & EG_FLAGS_IN_SHUTDOWN) != 0); + } + efree(data); + + return 0; +} + +static int frankenphp_task_stream_cast(php_stream *stream, int castas, + void **ret) { + if (castas != PHP_STREAM_AS_FD_FOR_SELECT) { + return FAILURE; + } + if (ret != NULL) { + frankenphp_task_stream_data *data = stream->abstract; + *(php_socket_t *)ret = (php_socket_t)data->fd; + } + + return SUCCESS; +} + +static int frankenphp_task_stream_set_option(php_stream *stream, int option, + int value, void *ptrparam) { + (void)value; + frankenphp_task_stream_data *data = stream->abstract; + switch (option) { + case PHP_STREAM_OPTION_READ_TIMEOUT: { + struct timeval *tv = ptrparam; + data->timeout_ms = + tv->tv_sec < 0 ? -1 : (int)(tv->tv_sec * 1000 + tv->tv_usec / 1000); + + return PHP_STREAM_OPTION_RETURN_OK; + } + case PHP_STREAM_OPTION_CHECK_LIVENESS: + /* feof(): the other side closed its stream */ + return go_frankenphp_task_side_gone(data->task, data->sender) + ? PHP_STREAM_OPTION_RETURN_ERR + : PHP_STREAM_OPTION_RETURN_OK; + case PHP_STREAM_OPTION_META_DATA_API: + add_assoc_bool((zval *)ptrparam, "timed_out", data->timed_out); + add_assoc_bool((zval *)ptrparam, "blocked", 1); + add_assoc_bool((zval *)ptrparam, "eof", stream->eof); + + return PHP_STREAM_OPTION_RETURN_OK; + default: + return PHP_STREAM_OPTION_RETURN_NOTIMPL; + } +} + +#define FRANKENPHP_TASK_STREAM_OPS(label) \ + { \ + frankenphp_task_stream_write, frankenphp_task_stream_read, \ + frankenphp_task_stream_close, NULL, label, NULL, \ + frankenphp_task_stream_cast, NULL, frankenphp_task_stream_set_option \ + } +static const php_stream_ops frankenphp_task_sender_ops = + FRANKENPHP_TASK_STREAM_OPS("FrankenPHP task sender"); +static const php_stream_ops frankenphp_task_receiver_ops = + FRANKENPHP_TASK_STREAM_OPS("FrankenPHP task receiver"); + +static php_stream *frankenphp_task_stream_open(uintptr_t task, intptr_t fd, + bool sender) { + frankenphp_task_stream_data *data = ecalloc(1, sizeof(*data)); + data->task = task; + data->fd = fd; + data->timeout_ms = -1; + data->sender = sender; + + return php_stream_alloc(sender ? &frankenphp_task_sender_ops + : &frankenphp_task_receiver_ops, + data, NULL, "r"); +} + +/* Waits for a signal on the side's descriptor without consuming it: 1 when + * one is pending, 0 on timeout. Interrupted polls are retried, like PHP's + * own stream code does. */ +static int frankenphp_task_stream_poll(frankenphp_task_stream_data *data, + int timeout_ms) { + for (;;) { + int n = + php_pollfd_for_ms((php_socket_t)data->fd, PHP_POLLREADABLE, timeout_ms); + if (n < 0 && php_socket_errno() == EINTR) { + continue; + } + + return n > 0; + } +} + +/* Consumes the signal of an event the Go side reported, waiting for it if + * the other side has not written it yet: the state is set before the + * signal, so the wait is momentary, and one signal per event keeps + * stream_select() exact. */ +static void frankenphp_task_stream_consume(frankenphp_task_stream_data *data) { + while (!frankenphp_task_chan_consume(data->fd)) { + frankenphp_task_stream_poll(data, -1); + } +} + +PHP_FUNCTION(frankenphp_send_task) { + zend_string *name; + zval *payload; + double timeout = 30.0; + bool timeout_is_null = false; + ZEND_PARSE_PARAMETERS_START(2, 3) + Z_PARAM_STR(name) + Z_PARAM_ARRAY(payload) + Z_PARAM_OPTIONAL + Z_PARAM_DOUBLE_OR_NULL(timeout, timeout_is_null) + ZEND_PARSE_PARAMETERS_END(); + + if (!timeout_is_null && (zend_isnan(timeout) || timeout < 0)) { + zend_argument_value_error(3, "must be greater than or equal to 0"); + RETURN_THROWS(); + } + /* past what a duration holds, infinity included, waits forever like null */ + int timeout_ms = -1; + if (!timeout_is_null && timeout * 1000 < (double)INT_MAX) { + timeout_ms = (int)(timeout * 1000); + } + if (!persistent_zval_validate(payload)) { + zend_value_error( + "frankenphp_send_task(): payload values must be null, " + "scalars, arrays or enums, nested no deeper than %d levels", + PERSISTENT_ZVAL_MAX_DEPTH); + RETURN_THROWS(); + } + + zval persistent; + persistent_zval_persist(&persistent, payload); + + /* the Go side owns the payload from here on, it frees it on failure */ + struct go_frankenphp_send_task_return task = + go_frankenphp_send_task(frankenphp_thread_index(), ZSTR_VAL(name), + ZSTR_LEN(name), Z_ARRVAL(persistent)); + if (task.r2 != NULL) { + zend_throw_exception(spl_ce_RuntimeException, task.r2, 0); + free(task.r2); + RETURN_THROWS(); + } + + /* the task is queued from here on: a bailout (memory limit) must not + * leave the sender's side open, the receiver would wait on it forever */ + php_stream *stream = NULL; + zend_try { stream = frankenphp_task_stream_open(task.r0, task.r1, true); } + zend_catch { + go_frankenphp_task_cancel(task.r0, false); + go_frankenphp_task_sender_close(task.r0); + zend_bailout(); + } + zend_end_try(); + + /* wait for the pickup in the kernel: the thread taking the task signals + * the sender's side, so does the Go side when the wait must end without a + * pickup, see go_frankenphp_send_task */ + frankenphp_task_stream_data *data = stream->abstract; + for (;;) { + if (!frankenphp_task_stream_poll(data, timeout_ms)) { + /* nobody took the task in time, unless right now */ + if (go_frankenphp_task_cancel(task.r0, true)) { + php_stream_close(stream); + zend_throw_exception_ex(spl_ce_RuntimeException, 0, + "frankenphp_send_task(): no thread of " + "background worker \"%s\" picked up the " + "task in time", + ZSTR_VAL(name)); + RETURN_THROWS(); + } + frankenphp_task_stream_consume(data); + + break; + } + + struct go_frankenphp_task_await_return state = + go_frankenphp_task_await(task.r0); + if (state.r0 == 0) { + /* a signal ahead of its event, or a stale one */ + frankenphp_task_chan_consume(data->fd); + continue; + } + frankenphp_task_stream_consume(data); + if (state.r0 == 1) { + break; + } + go_frankenphp_task_cancel(task.r0, false); + php_stream_close(stream); + zend_throw_exception(spl_ce_RuntimeException, state.r1, 0); + free(state.r1); + RETURN_THROWS(); + } + + php_stream_to_zval(stream, return_value); +} + +PHP_FUNCTION(frankenphp_read_task) { + zval *zstream; + ZEND_PARSE_PARAMETERS_START(1, 1) + Z_PARAM_RESOURCE(zstream) + ZEND_PARSE_PARAMETERS_END(); + + php_stream *stream; + php_stream_from_zval(stream, zstream); + if (stream->ops != &frankenphp_task_sender_ops) { + zend_argument_type_error( + 1, "must be a stream returned by frankenphp_send_task()"); + RETURN_THROWS(); + } + frankenphp_task_stream_data *data = stream->abstract; + + for (;;) { + struct go_frankenphp_read_task_return update = + go_frankenphp_read_task(data->task); + switch (update.r1) { + case FRANKENPHP_TASK_READ_UPDATE: + frankenphp_task_stream_consume(data); + zend_try { frankenphp_vars_to_request(return_value, update.r0); } + zend_catch { + frankenphp_vars_free(update.r0); + zend_bailout(); + } + zend_end_try(); + frankenphp_vars_free(update.r0); + return; + case FRANKENPHP_TASK_READ_COMPLETED: + case FRANKENPHP_TASK_READ_ABORTED: + if (!data->settled) { + data->settled = true; + frankenphp_task_stream_consume(data); + stream->eof = 1; + } + if (update.r1 == FRANKENPHP_TASK_READ_COMPLETED) { + RETURN_NULL(); + } + zend_throw_exception(spl_ce_RuntimeException, + "frankenphp_read_task(): the background worker " + "exited without completing the task", + 0); + RETURN_THROWS(); + default: + /* nothing yet: wait for the next signal, without consuming it, the + * event it announces does */ + if (!frankenphp_task_stream_poll(data, data->timeout_ms)) { + data->timed_out = true; + zend_throw_exception( + spl_ce_RuntimeException, + "frankenphp_read_task(): timed out waiting for the next update", 0); + RETURN_THROWS(); + } + } + } +} + +PHP_FUNCTION(frankenphp_receive_task) { + ZEND_PARSE_PARAMETERS_NONE(); + + if (!is_background_worker) { + zend_throw_exception( + spl_ce_RuntimeException, + "frankenphp_receive_task() can only be called from a background worker", + 0); + RETURN_THROWS(); + } + + struct go_frankenphp_receive_task_return task = + go_frankenphp_receive_task(frankenphp_thread_index()); + if (task.r0 == 0) { + RETURN_NULL(); + } + + /* the task is this thread's from here on: a bailout while copying the + * payload (memory limit, a fatal error in an autoloader) or creating the + * stream must not leave it open, the sender would wait on it forever */ + zval payload; + php_stream *stream = NULL; + zend_try { + frankenphp_vars_to_request(&payload, task.r1); + if (!EG(exception)) { + stream = frankenphp_task_stream_open(task.r0, task.r2, false); + } + } + zend_catch { + frankenphp_vars_free(task.r1); + go_frankenphp_task_receiver_close(task.r0, true); + zend_bailout(); + } + zend_end_try(); + frankenphp_vars_free(task.r1); + + if (EG(exception)) { + /* an enum of the payload does not resolve here: the task cannot be + * processed, the sender is told so */ + zval_ptr_dtor(&payload); + go_frankenphp_task_receiver_close(task.r0, true); + RETURN_THROWS(); + } + + zval zstream; + php_stream_to_zval(stream, &zstream); + array_init_size(return_value, 2); + add_next_index_zval(return_value, &zstream); + add_next_index_zval(return_value, &payload); +} + +PHP_FUNCTION(frankenphp_update_task) { + zval *zstream, *data; + ZEND_PARSE_PARAMETERS_START(2, 2) + Z_PARAM_RESOURCE(zstream) + Z_PARAM_ARRAY(data) + ZEND_PARSE_PARAMETERS_END(); + + php_stream *stream; + php_stream_from_zval(stream, zstream); + if (stream->ops != &frankenphp_task_receiver_ops) { + zend_argument_type_error( + 1, "must be a stream returned by frankenphp_receive_task()"); + RETURN_THROWS(); + } + if (!persistent_zval_validate(data)) { + zend_value_error("frankenphp_update_task(): values must be null, scalars, " + "arrays or enums, nested no deeper than %d levels", + PERSISTENT_ZVAL_MAX_DEPTH); + RETURN_THROWS(); + } + + zval persistent; + persistent_zval_persist(&persistent, data); + + /* the Go side owns the update from here on, it frees it on failure */ + char *error = go_frankenphp_update_task( + ((frankenphp_task_stream_data *)stream->abstract)->task, + Z_ARRVAL(persistent)); + if (error != NULL) { + zend_throw_exception(spl_ce_RuntimeException, error, 0); + free(error); + RETURN_THROWS(); + } +} + /* {{{ thread-safe opcache reset */ PHP_FUNCTION(frankenphp_opcache_reset) { go_schedule_opcache_reset(frankenphp_thread_index()); @@ -1075,6 +1936,10 @@ static const zend_function_entry frankenphp_test_hook_functions[] = { #endif PHP_MINIT_FUNCTION(frankenphp) { + frankenphp_worker_handle_ops = php_stream_socket_ops; + frankenphp_worker_handle_ops.label = "FrankenPHP worker handle"; + frankenphp_worker_handle_ops.close = frankenphp_worker_handle_close; + register_frankenphp_symbols(module_number); #ifndef PHP_WIN32 /* MINIT runs once per ZTS thread — guard the atfork registration */ @@ -1598,6 +2463,15 @@ static void *php_thread(void *arg) { } zend_end_try(); + /* The stop socket of a background worker is plain thread-local state that + * frankenphp_update_local_thread_context() only releases on recycle: close + * it here too so it does not outlive the thread on shutdown, reboot or an + * unhealthy exit. The Go side's end is closed by the Go side. */ + if (is_background_worker) { + is_background_worker = false; + frankenphp_worker_close_stop_socks(); + } + /* Must precede ts_free_thread: that frees the TSRM storage backing * the slot's &EG() pointers. Clearing first means any concurrent * force-kill either ran before us or sees a zero slot. */ diff --git a/frankenphp.go b/frankenphp.go index 3f7bbdf582..b8928eed59 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -172,26 +172,42 @@ func checkPHPConfig(config PHPConfig) error { return nil } -func calculateMaxThreads(opt *opt) (numWorkers int, _ error) { +// calculateMaxThreads resolves num_threads and max_threads against the HTTP +// workers and returns their thread count, plus the threads of the background +// workers, which take no part in that budget: they come on top of it +func calculateMaxThreads(opt *opt) (numWorkers, backgroundThreads int, _ error) { maxProcs := runtime.GOMAXPROCS(0) * 2 maxThreadsFromWorkers := 0 for i, w := range opt.workers { + if w.isBackgroundWorker { + if w.num < 1 { + name := w.name + if name == "" { + name = w.fileName + } + + return 0, 0, fmt.Errorf("background worker %q must declare num >= 1", name) + } + backgroundThreads += w.num + + continue + } + if w.num <= 0 { // https://github.com/php/frankenphp/issues/126 opt.workers[i].num = maxProcs } - metrics.TotalWorkers(w.name, w.num) numWorkers += opt.workers[i].num if w.maxThreads > 0 { if w.maxThreads < w.num { - return 0, fmt.Errorf("worker max_threads (%d) must be greater or equal to worker num (%d) (%q)", w.maxThreads, w.num, w.fileName) + return 0, 0, fmt.Errorf("worker max_threads (%d) must be greater or equal to worker num (%d) (%q)", w.maxThreads, w.num, w.fileName) } if w.maxThreads > opt.maxThreads && opt.maxThreads > 0 { - return 0, fmt.Errorf("worker max_threads (%d) cannot be greater than total max_threads (%d) (%q)", w.maxThreads, opt.maxThreads, w.fileName) + return 0, 0, fmt.Errorf("worker max_threads (%d) cannot be greater than total max_threads (%d) (%q)", w.maxThreads, opt.maxThreads, w.fileName) } maxThreadsFromWorkers += w.maxThreads - w.num @@ -215,19 +231,19 @@ func calculateMaxThreads(opt *opt) (numWorkers int, _ error) { if numThreadsIsSet && !maxThreadsIsSet { opt.maxThreads = opt.numThreads if opt.numThreads <= numWorkers { - return 0, fmt.Errorf("num_threads (%d) must be greater than the number of worker threads (%d)", opt.numThreads, numWorkers) + return 0, 0, fmt.Errorf("num_threads (%d) must be greater than the number of worker threads (%d)", opt.numThreads, numWorkers) } - return numWorkers, nil + return numWorkers, backgroundThreads, nil } if maxThreadsIsSet && !numThreadsIsSet { opt.numThreads = numWorkers + 1 if !maxThreadsIsAuto && opt.numThreads > opt.maxThreads { - return 0, fmt.Errorf("max_threads (%d) must be greater than the number of worker threads (%d)", opt.maxThreads, numWorkers) + return 0, 0, fmt.Errorf("max_threads (%d) must be greater than the number of worker threads (%d)", opt.maxThreads, numWorkers) } - return numWorkers, nil + return numWorkers, backgroundThreads, nil } if !numThreadsIsSet { @@ -239,19 +255,19 @@ func calculateMaxThreads(opt *opt) (numWorkers int, _ error) { } opt.maxThreads = opt.numThreads - return numWorkers, nil + return numWorkers, backgroundThreads, nil } // both num_threads and max_threads are set if opt.numThreads <= numWorkers { - return 0, fmt.Errorf("num_threads (%d) must be greater than the number of worker threads (%d)", opt.numThreads, numWorkers) + return 0, 0, fmt.Errorf("num_threads (%d) must be greater than the number of worker threads (%d)", opt.numThreads, numWorkers) } if !maxThreadsIsAuto && opt.maxThreads < opt.numThreads { - return 0, fmt.Errorf("max_threads (%d) must be greater than or equal to num_threads (%d)", opt.maxThreads, opt.numThreads) + return 0, 0, fmt.Errorf("max_threads (%d) must be greater than or equal to num_threads (%d)", opt.maxThreads, opt.numThreads) } - return numWorkers, nil + return numWorkers, backgroundThreads, nil } // Init starts the PHP runtime and the configured workers. @@ -300,13 +316,15 @@ func Init(options ...Option) error { registerServers(opt.servers) - workerThreadCount, err := calculateMaxThreads(opt) + workerThreadCount, backgroundThreads, err := calculateMaxThreads(opt) if err != nil { shutdown() return err } - metrics.TotalThreads(opt.numThreads) + // background workers run on threads of their own, on top of the budget + // num_threads and max_threads describe for HTTP traffic + metrics.TotalThreads(opt.numThreads + backgroundThreads) config := Config() @@ -324,13 +342,23 @@ func Init(options ...Option) error { } } else { opt.numThreads = 1 + if workerThreadCount > 1 || backgroundThreads > 0 { + shutdown() + return fmt.Errorf("%d worker threads are declared, but this PHP build is not ZTS and runs a single thread", workerThreadCount+backgroundThreads) + } if globalLogger.Enabled(globalCtx, slog.LevelWarn) { globalLogger.LogAttrs(globalCtx, slog.LevelWarn, `ZTS is not enabled, only 1 thread will be available, recompile PHP using the "--enable-zts" configuration option or performance will be degraded`) } } - mainThread, err := initPHPThreads(opt.numThreads, opt.maxThreads, opt.phpIni) + maxThreads := opt.maxThreads + if maxThreads > 0 { + maxThreads += backgroundThreads + } + // in auto mode (maxThreads < 0), the main thread adds the reservation to + // the limit it resolves, see setAutomaticMaxThreads() + mainThread, err := initPHPThreads(opt.numThreads+backgroundThreads, maxThreads, backgroundThreads, opt.phpIni) if err != nil { shutdown() return err @@ -484,7 +512,7 @@ func go_apache_request_headers(threadIndex C.uintptr_t) (*C.go_string, C.size_t) // worker mode, not handling a request if fc.logger.Enabled(fc.ctx, slog.LevelDebug) { - fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "apache_request_headers() called in non-HTTP context", slog.String("worker", fc.worker.name)) + fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "apache_request_headers() called in non-HTTP context", slog.String("worker", fc.worker.qualifiedName)) } return nil, 0 @@ -824,8 +852,6 @@ func resetGlobals() { globalCtx = context.Background() globalLogger = slog.Default() workers = nil - workersByName = nil - globalWorkersByPath = nil servers = nil watcherIsEnabled = false maxIdleTime = defaultMaxIdleTime diff --git a/frankenphp.h b/frankenphp.h index 99ac0ab7ec..407804a174 100644 --- a/frankenphp.h +++ b/frankenphp.h @@ -201,6 +201,26 @@ size_t frankenphp_get_thread_memory_usage(uintptr_t thread_index); void frankenphp_force_kill_thread(force_kill_slot slot); void frankenphp_release_thread_for_kill(force_kill_slot slot); +/* Background worker primitives. */ +intptr_t frankenphp_set_background_worker_and_get_stop_sock(void); +void frankenphp_close_sock(intptr_t s); +void frankenphp_worker_close_stop_sock(intptr_t s); +void frankenphp_worker_signal_task(intptr_t s); +int frankenphp_task_chan_open(intptr_t fds[2]); +void frankenphp_task_chan_signal(intptr_t fd0, intptr_t fd1, int side); +bool frankenphp_task_chan_consume(intptr_t fd); +void frankenphp_task_chan_drain(intptr_t fd); +void frankenphp_vars_to_request(zval *return_value, HashTable *table); +void frankenphp_vars_free(HashTable *table); + +/* Results of go_frankenphp_read_task. */ +enum { + FRANKENPHP_TASK_READ_UPDATE, + FRANKENPHP_TASK_READ_COMPLETED, + FRANKENPHP_TASK_READ_ABORTED, + FRANKENPHP_TASK_READ_PENDING, +}; + void register_extensions(zend_module_entry **m, int len); #endif diff --git a/frankenphp.stub.php b/frankenphp.stub.php index d6c85aa05f..c131a7ba7c 100644 --- a/frankenphp.stub.php +++ b/frankenphp.stub.php @@ -54,3 +54,94 @@ function mercure_publish(string|array $topics, string $data = '', bool $private * array $context Values of the array will be converted to the corresponding Go type (if supported by FrankenPHP) and added to the context of the structured logs using https://pkg.go.dev/log/slog#Attr */ function frankenphp_log(string $message, int $level = 0, array $context = []): void {} + +/** + * EXPERIMENTAL: returns the handle of the current background worker, a + * stream to wait on, alone or with the script's own streams: it becomes + * readable when FrankenPHP needs the script's attention, its drain + * included, and frankenphp_worker_tick() then tells whether the worker + * still runs. Every call of a run returns the same stream, a fresh one over + * the same socket once the script closed it. Only callable from inside a + * background worker. + * + * @return resource + */ +function frankenphp_get_worker_handle() {} + +/** + * EXPERIMENTAL: the ready point and liveness check of a background worker, + * the background analog of frankenphp_handle_request(). The first call of a + * run marks the worker ready: the server start waits for it, and an exit + * before it counts as a failure. max_execution_time applies until that + * call and not after. It returns false once FrankenPHP drains the + * worker, on shutdown, reboot or restart, so the script can leave its loop, + * and true otherwise. It never blocks and never hands out work: the script + * waits on the stream returned by frankenphp_get_worker_handle() and calls + * this when it is readable. Whatever FrankenPHP wrote on that stream is + * consumed here, the script does not have to read it. Only callable from + * inside a background worker. + */ +function frankenphp_worker_tick(): bool {} + +/** + * Publishes the vars of the current background worker: the array replaces + * the previous snapshot, atomically for readers, which get copies. Values + * must be null, scalars, arrays or enums. Only callable from inside a + * background worker. + */ +function frankenphp_set_vars(array $vars): void {} + +/** + * Returns a copy of the vars last published by the named background worker, + * resolved within the current php_server, then among global workers. Blocks + * until that worker reached its ready point. Throws if the worker is + * unknown, if it is ready but has not published any vars, or if background + * workers wait on each other in a cycle. + */ +function frankenphp_get_vars(string $name): array {} + +/** + * Hands a task to the named background worker, resolved like + * frankenphp_get_vars() does, and returns a stream carrying the updates it + * sends back. Blocks until a thread of the worker picks the task up; throws + * if none did within $timeout seconds (null waits forever), or if the + * worker is unknown. Payload values must be null, scalars, arrays or enums. + * Closing the stream abandons the task. + * + * @return resource + */ +function frankenphp_send_task(string $name, array $payload, ?float $timeout = 30.0) {} + +/** + * Returns the next update of a task, blocking until the background worker + * sends one, or null once it completed the task. Throws if the worker ended + * its script with the task open. The stream also works with stream_select(). + * + * @param resource $stream A stream returned by frankenphp_send_task() + */ +function frankenphp_read_task($stream): ?array {} + +/** + * Dequeues a task sent to the current background worker, without blocking: + * [$stream, $payload], or null when there is none. Each task sent wakes one + * thread of the worker through its handle, so a script calls this after + * frankenphp_worker_tick() returned. A wake-up is not a count, and null + * after one is expected in a pool. $stream reaches EOF when the sender + * closes its own stream, for stream_select() and feof(). Only callable from + * inside a background worker. + * + * @return array{resource, array}|null + */ +function frankenphp_receive_task(): ?array {} + +/** + * Sends an update, progress or result, to the sender of a task; fclose() on + * the stream completes the task, before the script ends: a close during + * request shutdown, from a destructor or a shutdown function included, + * reports the task as not completed instead. Values must be null, scalars, + * arrays or enums. At most 16 updates are buffered: past that, blocks until + * the sender reads. Throws once the sender closed its stream. + * + * @param resource $stream A stream returned by frankenphp_receive_task() + */ +function frankenphp_update_task($stream, array $data): void {} diff --git a/frankenphp_arginfo.h b/frankenphp_arginfo.h index 4f2707cbca..010b607208 100644 --- a/frankenphp_arginfo.h +++ b/frankenphp_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 60f0d27c04f94d7b24c052e91ef294595a2bc421 */ + * Stub hash: d74aff3740ccd178824fffa034813e0eb2d4dc51 */ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_handle_request, 0, 1, _IS_BOOL, 0) ZEND_ARG_TYPE_INFO(0, callback, IS_CALLABLE, 0) @@ -41,6 +41,36 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_log, 0, 1, IS_VOID, 0 ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, context, IS_ARRAY, 0, "[]") ZEND_END_ARG_INFO() +ZEND_BEGIN_ARG_INFO_EX(arginfo_frankenphp_get_worker_handle, 0, 0, 0) +ZEND_END_ARG_INFO() + +#define arginfo_frankenphp_worker_tick arginfo_frankenphp_finish_request + +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_set_vars, 0, 1, IS_VOID, 0) + ZEND_ARG_TYPE_INFO(0, vars, IS_ARRAY, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_get_vars, 0, 1, IS_ARRAY, 0) + ZEND_ARG_TYPE_INFO(0, name, IS_STRING, 0) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_INFO_EX(arginfo_frankenphp_send_task, 0, 0, 2) + ZEND_ARG_TYPE_INFO(0, name, IS_STRING, 0) + ZEND_ARG_TYPE_INFO(0, payload, IS_ARRAY, 0) + ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, timeout, IS_DOUBLE, 1, "30.0") +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_read_task, 0, 1, IS_ARRAY, 1) + ZEND_ARG_INFO(0, stream) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_receive_task, 0, 0, IS_ARRAY, 1) +ZEND_END_ARG_INFO() + +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_frankenphp_update_task, 0, 2, IS_VOID, 0) + ZEND_ARG_INFO(0, stream) + ZEND_ARG_TYPE_INFO(0, data, IS_ARRAY, 0) +ZEND_END_ARG_INFO() ZEND_FUNCTION(frankenphp_handle_request); ZEND_FUNCTION(headers_send); @@ -49,7 +79,14 @@ ZEND_FUNCTION(frankenphp_request_headers); ZEND_FUNCTION(frankenphp_response_headers); ZEND_FUNCTION(mercure_publish); ZEND_FUNCTION(frankenphp_log); - +ZEND_FUNCTION(frankenphp_get_worker_handle); +ZEND_FUNCTION(frankenphp_worker_tick); +ZEND_FUNCTION(frankenphp_set_vars); +ZEND_FUNCTION(frankenphp_get_vars); +ZEND_FUNCTION(frankenphp_send_task); +ZEND_FUNCTION(frankenphp_read_task); +ZEND_FUNCTION(frankenphp_receive_task); +ZEND_FUNCTION(frankenphp_update_task); static const zend_function_entry ext_functions[] = { ZEND_FE(frankenphp_handle_request, arginfo_frankenphp_handle_request) @@ -63,6 +100,14 @@ static const zend_function_entry ext_functions[] = { ZEND_FALIAS(apache_response_headers, frankenphp_response_headers, arginfo_apache_response_headers) ZEND_FE(mercure_publish, arginfo_mercure_publish) ZEND_FE(frankenphp_log, arginfo_frankenphp_log) + ZEND_FE(frankenphp_get_worker_handle, arginfo_frankenphp_get_worker_handle) + ZEND_FE(frankenphp_worker_tick, arginfo_frankenphp_worker_tick) + ZEND_FE(frankenphp_set_vars, arginfo_frankenphp_set_vars) + ZEND_FE(frankenphp_get_vars, arginfo_frankenphp_get_vars) + ZEND_FE(frankenphp_send_task, arginfo_frankenphp_send_task) + ZEND_FE(frankenphp_read_task, arginfo_frankenphp_read_task) + ZEND_FE(frankenphp_receive_task, arginfo_frankenphp_receive_task) + ZEND_FE(frankenphp_update_task, arginfo_frankenphp_update_task) ZEND_FE_END }; diff --git a/metrics.go b/metrics.go index fc25816506..879a090816 100644 --- a/metrics.go +++ b/metrics.go @@ -11,11 +11,21 @@ import ( const ( StopReasonCrash = iota StopReasonRestart - StopReasonBootFailure // worker crashed before reaching frankenphp_handle_request + StopReasonBootFailure // worker exited before reaching its ready point: frankenphp_handle_request, or frankenphp_worker_tick for background workers ) type StopReason int +// TaskOutcome is how a task sent to a background worker ended +type TaskOutcome string + +const ( + TaskOutcomeCompleted TaskOutcome = "completed" // the worker closed the task's stream + TaskOutcomeAborted TaskOutcome = "aborted" // the worker's script ended with the task open + TaskOutcomeAbandoned TaskOutcome = "abandoned" // the sender closed its stream first + TaskOutcomeTimeout TaskOutcome = "timeout" // no thread picked the task up in time +) + type Metrics interface { // StartWorker collects started workers StartWorker(name string) @@ -40,6 +50,12 @@ type Metrics interface { DequeuedWorkerRequest(name string) QueuedRequest() DequeuedRequest() + // StartWorkerTask collects tasks picked up by a thread of a background worker + StartWorkerTask(name string) + // StopWorkerTask collects tasks a thread of a background worker is done with + StopWorkerTask(name string, duration time.Duration) + // WorkerTaskOutcome collects how tasks sent to a background worker ended + WorkerTaskOutcome(name string, outcome TaskOutcome) } type nullMetrics struct{} @@ -81,6 +97,12 @@ func (n nullMetrics) DequeuedWorkerRequest(string) {} func (n nullMetrics) QueuedRequest() {} func (n nullMetrics) DequeuedRequest() {} +func (n nullMetrics) StartWorkerTask(string) {} + +func (n nullMetrics) StopWorkerTask(string, time.Duration) {} + +func (n nullMetrics) WorkerTaskOutcome(string, TaskOutcome) {} + type PrometheusMetrics struct { registry prometheus.Registerer totalThreads prometheus.Gauge @@ -93,6 +115,8 @@ type PrometheusMetrics struct { workerRequestTime *prometheus.CounterVec workerRequestCount *prometheus.CounterVec workerQueueDepth *prometheus.GaugeVec + workerTaskCount *prometheus.CounterVec + workerTaskTime *prometheus.CounterVec queueDepth prometheus.Gauge mu sync.RWMutex } @@ -144,7 +168,7 @@ func (m *PrometheusMetrics) StopWorker(name string, reason StopReason) { m.totalWorkers.WithLabelValues(name).Dec() - // only decrement readyWorkers if the worker actually reached frankenphp_handle_request + // only decrement readyWorkers if the worker actually reached its ready point if reason != StopReasonBootFailure { m.readyWorkers.WithLabelValues(name).Dec() } @@ -177,7 +201,7 @@ func (m *PrometheusMetrics) TotalWorkers(string, int) { m.readyWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: ns, Name: "ready_workers", - Help: "Running workers that have successfully called frankenphp_handle_request at least once", + Help: "Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, frankenphp_worker_tick for background workers", }, basicLabels) m.mustRegister(m.readyWorkers) } @@ -186,7 +210,7 @@ func (m *PrometheusMetrics) TotalWorkers(string, int) { m.busyWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: ns, Name: "busy_workers", - Help: "Number of busy PHP workers for this worker", + Help: "Number of busy PHP workers for this worker: processing a request, or a task for a background worker", }, basicLabels) m.mustRegister(m.busyWorkers) } @@ -234,9 +258,30 @@ func (m *PrometheusMetrics) TotalWorkers(string, int) { Namespace: "frankenphp", Subsystem: sub, Name: "queue_depth", + Help: "Number of queued requests for this worker, or of tasks waiting for a thread of a background worker", }, basicLabels) m.mustRegister(m.workerQueueDepth) } + + if m.workerTaskCount == nil { + m.workerTaskCount = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: ns, + Subsystem: sub, + Name: "task_count", + Help: "Number of tasks sent to this background worker, by outcome: completed, aborted (the script ended with the task open), abandoned (the sender closed its stream first) or timeout (no thread picked the task up in time)", + }, []string{"worker", "outcome"}) + m.mustRegister(m.workerTaskCount) + } + + if m.workerTaskTime == nil { + m.workerTaskTime = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: ns, + Subsystem: sub, + Name: "task_time", + Help: "Time spent on tasks by all threads of this background worker, from pickup to the close of the task's stream", + }, basicLabels) + m.mustRegister(m.workerTaskTime) + } } func (m *PrometheusMetrics) TotalThreads(num int) { @@ -317,6 +362,37 @@ func (m *PrometheusMetrics) DequeuedRequest() { m.queueDepth.Dec() } +func (m *PrometheusMetrics) StartWorkerTask(name string) { + m.mu.RLock() + defer m.mu.RUnlock() + + if m.busyWorkers == nil { + return + } + m.busyWorkers.WithLabelValues(name).Inc() +} + +func (m *PrometheusMetrics) StopWorkerTask(name string, duration time.Duration) { + m.mu.RLock() + defer m.mu.RUnlock() + + if m.workerTaskTime == nil { + return + } + m.busyWorkers.WithLabelValues(name).Dec() + m.workerTaskTime.WithLabelValues(name).Add(duration.Seconds()) +} + +func (m *PrometheusMetrics) WorkerTaskOutcome(name string, outcome TaskOutcome) { + m.mu.RLock() + defer m.mu.RUnlock() + + if m.workerTaskCount == nil { + return + } + m.workerTaskCount.WithLabelValues(name, string(outcome)).Inc() +} + func (m *PrometheusMetrics) Shutdown() { m.mu.Lock() defer m.mu.Unlock() @@ -356,6 +432,14 @@ func (m *PrometheusMetrics) Shutdown() { if m.workerQueueDepth != nil { m.registry.Unregister(m.workerQueueDepth) } + + if m.workerTaskCount != nil { + m.registry.Unregister(m.workerTaskCount) + } + + if m.workerTaskTime != nil { + m.registry.Unregister(m.workerTaskTime) + } } func NewPrometheusMetrics(registry prometheus.Registerer) *PrometheusMetrics { @@ -385,6 +469,8 @@ func NewPrometheusMetrics(registry prometheus.Registerer) *PrometheusMetrics { workerCrashes: nil, readyWorkers: nil, workerQueueDepth: nil, + workerTaskCount: nil, + workerTaskTime: nil, } m.mustRegister(m.totalThreads) diff --git a/metrics_test.go b/metrics_test.go index 846a926569..ea48b52330 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -50,6 +50,8 @@ func TestPrometheusMetrics_TotalWorkers(t *testing.T) { require.Nil(t, m.workerRestarts) require.Nil(t, m.workerRequestTime) require.Nil(t, m.workerRequestCount) + require.Nil(t, m.workerTaskCount) + require.Nil(t, m.workerTaskTime) m.TotalWorkers("test_worker", 2) @@ -60,6 +62,65 @@ func TestPrometheusMetrics_TotalWorkers(t *testing.T) { require.NotNil(t, m.workerRestarts) require.NotNil(t, m.workerRequestTime) require.NotNil(t, m.workerRequestCount) + require.NotNil(t, m.workerTaskCount) + require.NotNil(t, m.workerTaskTime) +} + +func TestPrometheusMetrics_WorkerTask(t *testing.T) { + m := createPrometheusMetrics() + m.TotalWorkers("bg_worker", 1) + m.StartWorkerTask("bg_worker") + m.StopWorkerTask("bg_worker", 3*time.Second) + m.WorkerTaskOutcome("bg_worker", TaskOutcomeCompleted) + m.WorkerTaskOutcome("bg_worker", TaskOutcomeTimeout) + + inputs := []struct { + name string + c prometheus.Collector + metadata string + expect string + }{ + { + name: "Testing BusyWorkers", + c: m.busyWorkers, + metadata: ` + # HELP frankenphp_busy_workers Number of busy PHP workers for this worker: processing a request, or a task for a background worker + # TYPE frankenphp_busy_workers gauge + `, + expect: ` + frankenphp_busy_workers{worker="bg_worker"} 0 + `, + }, + { + name: "Testing WorkerTaskTime", + c: m.workerTaskTime, + metadata: ` + # HELP frankenphp_worker_task_time Time spent on tasks by all threads of this background worker, from pickup to the close of the task's stream + # TYPE frankenphp_worker_task_time counter + `, + expect: ` + frankenphp_worker_task_time{worker="bg_worker"} 3 + `, + }, + { + name: "Testing WorkerTaskCount", + c: m.workerTaskCount, + metadata: ` + # HELP frankenphp_worker_task_count Number of tasks sent to this background worker, by outcome: completed, aborted (the script ended with the task open), abandoned (the sender closed its stream first) or timeout (no thread picked the task up in time) + # TYPE frankenphp_worker_task_count counter + `, + expect: ` + frankenphp_worker_task_count{outcome="completed",worker="bg_worker"} 1 + frankenphp_worker_task_count{outcome="timeout",worker="bg_worker"} 1 + `, + }, + } + + for _, input := range inputs { + t.Run(input.name, func(t *testing.T) { + require.NoError(t, testutil.CollectAndCompare(input.c, strings.NewReader(input.metadata+input.expect))) + }) + } } func TestPrometheusMetrics_StopWorkerRequest(t *testing.T) { @@ -88,7 +149,7 @@ func TestPrometheusMetrics_StopWorkerRequest(t *testing.T) { name: "Testing BusyWorkers", c: m.busyWorkers, metadata: ` - # HELP frankenphp_busy_workers Number of busy PHP workers for this worker + # HELP frankenphp_busy_workers Number of busy PHP workers for this worker: processing a request, or a task for a background worker # TYPE frankenphp_busy_workers gauge `, expect: ` @@ -131,7 +192,7 @@ func TestPrometheusMetrics_StartWorkerRequest(t *testing.T) { name: "Testing BusyWorkers", c: m.busyWorkers, metadata: ` - # HELP frankenphp_busy_workers Number of busy PHP workers for this worker + # HELP frankenphp_busy_workers Number of busy PHP workers for this worker: processing a request, or a task for a background worker # TYPE frankenphp_busy_workers gauge `, expect: ` @@ -185,7 +246,7 @@ func TestPrometheusMetrics_TestStopReasonCrash(t *testing.T) { name: "Testing ReadyWorkers", c: m.readyWorkers, metadata: ` - # HELP frankenphp_ready_workers Running workers that have successfully called frankenphp_handle_request at least once + # HELP frankenphp_ready_workers Running workers that have reached their ready point at least once: frankenphp_handle_request for HTTP workers, frankenphp_worker_tick for background workers # TYPE frankenphp_ready_workers gauge `, expect: ` diff --git a/options.go b/options.go index e1eaeb7b55..930eb5d182 100644 --- a/options.go +++ b/options.go @@ -57,6 +57,7 @@ type workerOpt struct { onServerStartup func() onServerShutdown func() server *Server + isBackgroundWorker bool } // WithContext sets the main context to use. @@ -239,6 +240,20 @@ func WithWorkerServerScope(s *Server) WorkerOption { } } +// EXPERIMENTAL: WithWorkerBackground marks this worker as a background +// (non-HTTP) worker. Background workers run outside the request cycle: +// they share the PHP runtime with HTTP threads but never receive HTTP +// requests. The script can park on the stream returned by +// frankenphp_get_worker_handle(), which reaches EOF when FrankenPHP +// drains the worker, to exit gracefully on shutdown or restart. +func WithWorkerBackground() WorkerOption { + return func(w *workerOpt) error { + w.isBackgroundWorker = true + + return nil + } +} + // WithWorkerMaxFailures sets the maximum number of consecutive failures before panicking func WithWorkerMaxFailures(maxFailures int) WorkerOption { return func(w *workerOpt) error { diff --git a/phpmainthread.go b/phpmainthread.go index 5e19c0fc75..e3236b4281 100644 --- a/phpmainthread.go +++ b/phpmainthread.go @@ -20,12 +20,16 @@ import ( // represents the main PHP thread // the thread needs to keep running as long as all other threads are running type phpMainThread struct { - state *state.ThreadState - done chan struct{} - numThreads int - maxThreads int - phpIni map[string]string - isRebooting atomic.Bool + state *state.ThreadState + done chan struct{} + numThreads int + maxThreads int + // backgroundThreads is the part of numThreads and maxThreads that + // background workers reserve: it takes no part in the HTTP budget, + // see calculateMaxThreads() + backgroundThreads int + phpIni map[string]string + isRebooting atomic.Bool } var ( @@ -41,13 +45,14 @@ var ( // initPHPThreads starts the main PHP thread, // a fixed number of inactive PHP threads // and reserves a fixed number of possible PHP threads -func initPHPThreads(numThreads int, numMaxThreads int, phpIni map[string]string) (*phpMainThread, error) { +func initPHPThreads(numThreads int, numMaxThreads int, backgroundThreads int, phpIni map[string]string) (*phpMainThread, error) { mainThread = &phpMainThread{ - state: state.NewThreadState(), - done: make(chan struct{}), - numThreads: numThreads, - maxThreads: numMaxThreads, - phpIni: phpIni, + state: state.NewThreadState(), + done: make(chan struct{}), + numThreads: numThreads, + maxThreads: numMaxThreads, + backgroundThreads: backgroundThreads, + phpIni: phpIni, } // initialize the first thread @@ -105,6 +110,9 @@ func drainPHPThreads() { } doneWG.Wait() + // no PHP thread can read them anymore, and the engine is still up + freeWorkerVars() + freeTaskChans() mainThread.state.Set(state.Done) mainThread.state.WaitFor(state.Reserved) C.frankenphp_destroy_thread_metrics() @@ -161,6 +169,10 @@ func (mainThread *phpMainThread) rebootAllThreads() bool { for _, thread := range rebootingThreads { rebootWg.Go(func() { + // wake up handlers parked in a blocking C call (background + // workers' stream_select on the stop socket) so they can yield + // for the reboot without waiting for the force-kill below + thread.handler.drain() close(thread.drainChan) if thread.state.WaitForStateWithTimeout(rebootGracePeriod, state.YieldingForReboot) { return @@ -259,18 +271,21 @@ func go_frankenphp_main_thread_is_ready() { // max_threads = auto // setAutomaticMaxThreads estimates the amount of threads based on php.ini and system memory_limit // If unable to get the system's memory limit, simply double num_threads +// The estimate is an HTTP one, like an explicit max_threads: the threads +// background workers reserve are added to it rather than taken out of it func (mainThread *phpMainThread) setAutomaticMaxThreads() { if mainThread.maxThreads >= 0 { return } + httpThreads := mainThread.numThreads - mainThread.backgroundThreads perThreadMemoryLimit := int64(C.frankenphp_get_current_memory_limit()) totalSysMemory := memory.TotalSysMemory() if perThreadMemoryLimit <= 0 || totalSysMemory == 0 { - mainThread.maxThreads = mainThread.numThreads * 2 + mainThread.maxThreads = httpThreads*2 + mainThread.backgroundThreads return } maxAllowedThreads := totalSysMemory / uint64(perThreadMemoryLimit) - mainThread.maxThreads = int(maxAllowedThreads) + mainThread.maxThreads = int(maxAllowedThreads) + mainThread.backgroundThreads if globalLogger.Enabled(globalCtx, slog.LevelDebug) { globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "Automatic thread limit", slog.Int("perThreadMemoryLimitMB", int(perThreadMemoryLimit/1024/1024)), slog.Int("maxThreads", mainThread.maxThreads)) diff --git a/phpmainthread_test.go b/phpmainthread_test.go index 3ae65e68b9..8d78e25e74 100644 --- a/phpmainthread_test.go +++ b/phpmainthread_test.go @@ -26,7 +26,7 @@ func setupGlobals(t *testing.T) { } func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { - _, err := initPHPThreads(1, 1, nil) // boot 1 thread + _, err := initPHPThreads(1, 1, 0, nil) // boot 1 thread assert.NoError(t, err) assert.Len(t, phpThreads, 1) @@ -41,7 +41,7 @@ func TestStartAndStopTheMainThreadWithOneInactiveThread(t *testing.T) { func TestTransitionRegularThreadToWorkerThread(t *testing.T) { setupGlobals(t) - _, err := initPHPThreads(1, 1, nil) + _, err := initPHPThreads(1, 1, 0, nil) assert.NoError(t, err) // transition to regular thread @@ -66,7 +66,7 @@ func TestTransitionRegularThreadToWorkerThread(t *testing.T) { func TestTransitionAThreadBetween2DifferentWorkers(t *testing.T) { setupGlobals(t) - _, err := initPHPThreads(1, 1, nil) + _, err := initPHPThreads(1, 1, 0, nil) assert.NoError(t, err) firstWorker := getDummyWorker(t, "transition-worker-1.php") secondWorker := getDummyWorker(t, "transition-worker-2.php") @@ -228,7 +228,7 @@ func TestQueuedRequestSurvivesReload(t *testing.T) { func TestFinishBootingAWorkerScript(t *testing.T) { setupGlobals(t) - _, err := initPHPThreads(1, 1, nil) + _, err := initPHPThreads(1, 1, 0, nil) assert.NoError(t, err) // boot the worker @@ -252,28 +252,24 @@ func TestFinishBootingAWorkerScript(t *testing.T) { func TestReturnAnErrorIf2WorkersHaveTheSameFileName(t *testing.T) { resetGlobals() - workers = []*worker{} - workersByName = map[string]*worker{} - globalWorkersByPath = map[string]*worker{} - w, err1 := newWorker(workerOpt{fileName: testDataPath + "/index.php"}) - assert.NoError(t, err1) - workers = append(workers, w) - workersByName[w.name] = w - globalWorkersByPath[w.fileName] = w - _, err2 := newWorker(workerOpt{fileName: testDataPath + "/index.php"}) - assert.Error(t, err2, "two workers cannot have the same filename") + fallbackServer.resetWorkers() + w1, err := newWorker(workerOpt{fileName: testDataPath + "/index.php"}) + assert.NoError(t, err) + assert.NoError(t, fallbackServer.addWorker(w1)) + w2, err := newWorker(workerOpt{fileName: testDataPath + "/index.php", name: "other"}) + assert.NoError(t, err) + assert.ErrorContains(t, fallbackServer.addWorker(w2), "two global workers cannot have the same filename") } func TestReturnAnErrorIf2ModuleWorkersHaveTheSameName(t *testing.T) { resetGlobals() - workers = []*worker{} - workersByName = map[string]*worker{} - w, err1 := newWorker(workerOpt{fileName: testDataPath + "/index.php", name: "workername"}) - assert.NoError(t, err1) - workers = append(workers, w) - workersByName[w.name] = w - _, err2 := newWorker(workerOpt{fileName: testDataPath + "/hello.php", name: "workername"}) - assert.Error(t, err2, "two workers cannot have the same name") + fallbackServer.resetWorkers() + w1, err := newWorker(workerOpt{fileName: testDataPath + "/index.php", name: "workername"}) + assert.NoError(t, err) + assert.NoError(t, fallbackServer.addWorker(w1)) + w2, err := newWorker(workerOpt{fileName: testDataPath + "/hello.php", name: "workername"}) + assert.NoError(t, err) + assert.ErrorContains(t, fallbackServer.addWorker(w2), "two global workers cannot have the same name") } func getDummyWorker(t *testing.T, fileName string) *worker { @@ -315,9 +311,9 @@ func allPossibleTransitions(worker1Path string, worker2Path string) []func(*phpT thread.boot() } }, - func(thread *phpThread) { convertToWorkerThread(thread, globalWorkersByPath[worker1Path]) }, + func(thread *phpThread) { convertToWorkerThread(thread, fallbackServer.workersByPath[worker1Path]) }, convertToInactiveThread, - func(thread *phpThread) { convertToWorkerThread(thread, globalWorkersByPath[worker2Path]) }, + func(thread *phpThread) { convertToWorkerThread(thread, fallbackServer.workersByPath[worker2Path]) }, convertToInactiveThread, } } @@ -371,7 +367,7 @@ func TestCorrectThreadCalculation(t *testing.T) { func testThreadCalculation(t *testing.T, expectedNumThreads int, expectedMaxThreads int, o *opt) { t.Helper() - _, err := calculateMaxThreads(o) + _, _, err := calculateMaxThreads(o) assert.NoError(t, err, "no error should be returned") assert.Equal(t, expectedNumThreads, o.numThreads, "num_threads must be correct") assert.Equal(t, expectedMaxThreads, o.maxThreads, "max_threads must be correct") @@ -380,7 +376,7 @@ func testThreadCalculation(t *testing.T, expectedNumThreads int, expectedMaxThre func testThreadCalculationError(t *testing.T, o *opt) { t.Helper() - _, err := calculateMaxThreads(o) + _, _, err := calculateMaxThreads(o) assert.Error(t, err, "configuration must error") } @@ -389,7 +385,8 @@ func TestContextAndLoggerMustNotBeNil(t *testing.T) { assert.NotNil(t, log, "logger is defined if all threads are inactive") assert.NotNil(t, ctx, "context is defined if all threads are inactive") - fc := newContextFromMessage(nil, nil, nil, &worker{}) + // a worker always belongs to a server, see newWorker() + fc := newContextFromMessage(nil, nil, nil, &worker{server: fallbackServer}) assert.NotNil(t, fc.logger, "logger is defined for message context") assert.NotNil(t, fc.ctx, "context is defined for message context") @@ -398,7 +395,7 @@ func TestContextAndLoggerMustNotBeNil(t *testing.T) { assert.NotNil(t, fc.logger, "logger is defined for request context") assert.NotNil(t, fc.ctx, "context is defined for request context") - fc, _ = newWorkerDummyContext(&worker{}) + fc, _ = newWorkerDummyContext(&worker{server: fallbackServer}) assert.NotNil(t, fc.logger, "logger is defined for worker dummy context") assert.NotNil(t, fc.ctx, "context is defined for worker dummy context") } diff --git a/phpthread.go b/phpthread.go index 39ee24b5d4..4165da482a 100644 --- a/phpthread.go +++ b/phpthread.go @@ -39,11 +39,10 @@ type threadHandler interface { beforeScriptExecution() string afterScriptExecution(exitStatus int) frankenPHPContext() *frankenPHPContext - // drain is a hook called by drainWorkerThreads right before drainChan is - // closed. Handlers that need to wake up a thread parked in a blocking C - // call (e.g. by closing a stop pipe) plug their signal in here. All - // current handlers are no-ops; this is the seam later handler types use - // without having to modify drainWorkerThreads. + // drain is a hook called right before drainChan is closed on shutdown + // and reboot. Handlers that need to wake up a thread parked in a + // blocking C call (background workers' stream_select on the stop socket) + // plug their signal in here; the other handlers are no-ops. drain() } @@ -119,6 +118,9 @@ func (thread *phpThread) shutdown() { return } + // wake up handlers parked in a blocking C call (background workers' + // stream_select on the stop socket); no-op for the other handlers + thread.handler.drain() close(thread.drainChan) // Arm force-kill after the grace period to wake any thread stuck in @@ -157,6 +159,9 @@ func (thread *phpThread) setHandler(handler threadHandler) { return } + // wake up a handler parked in a blocking C call (background workers' + // stream_select on the stop socket) so it can yield for the transition + thread.handler.drain() close(thread.drainChan) thread.state.WaitFor(state.TransitionInProgress) diff --git a/requestoptions.go b/requestoptions.go index 962727562f..00c5892f05 100644 --- a/requestoptions.go +++ b/requestoptions.go @@ -2,6 +2,7 @@ package frankenphp import ( "errors" + "fmt" "log/slog" "net/http" "path/filepath" @@ -206,12 +207,22 @@ func WithRequestBodyTimeout(timeout time.Duration) RequestOption { } // WithWorkerName sets the worker that should handle the request +// the name is resolved among the workers of the request's server first, then among global workers func WithWorkerName(name string) RequestOption { return func(o *frankenPHPContext) error { - if name != "" { - o.worker = workersByName[name] + if name == "" { + return nil } + w := o.server.workersByName[name] + if w == nil { + w = fallbackServer.workersByName[name] + } + if w != nil && w.isBackgroundWorker { + return fmt.Errorf("background worker %q cannot handle requests", name) + } + o.worker = w + return nil } } diff --git a/scaling.go b/scaling.go index dd21a7e37c..c26465efbd 100644 --- a/scaling.go +++ b/scaling.go @@ -96,7 +96,7 @@ func scaleWorkerThread(worker *worker, done chan struct{}, mstate *state.ThreadS thread, err := addWorkerThread(worker) if err != nil { if globalLogger.Enabled(globalCtx, slog.LevelWarn) { - globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "could not increase max_threads, consider raising this limit", slog.String("worker", worker.name), slog.Any("error", err)) + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "could not increase max_threads, consider raising this limit", slog.String("worker", worker.qualifiedName), slog.Any("error", err)) } return @@ -105,7 +105,7 @@ func scaleWorkerThread(worker *worker, done chan struct{}, mstate *state.ThreadS autoScaledThreads = append(autoScaledThreads, thread) if globalLogger.Enabled(globalCtx, slog.LevelInfo) { - globalLogger.LogAttrs(globalCtx, slog.LevelInfo, "upscaling worker thread", slog.String("worker", worker.name), slog.Int("thread", thread.threadIndex), slog.Int("num_threads", len(autoScaledThreads))) + globalLogger.LogAttrs(globalCtx, slog.LevelInfo, "upscaling worker thread", slog.String("worker", worker.qualifiedName), slog.Int("thread", thread.threadIndex), slog.Int("num_threads", len(autoScaledThreads))) } } @@ -177,7 +177,7 @@ func startUpscalingThreads(maxScaledThreads int, scale chan *frankenPHPContext, // check for max worker threads here again in case requests overflowed while waiting if fc.worker.isAtThreadLimit() { if globalLogger.Enabled(globalCtx, slog.LevelInfo) { - globalLogger.LogAttrs(globalCtx, slog.LevelInfo, "cannot scale worker thread, max threads reached for worker", slog.String("worker", fc.worker.name)) + globalLogger.LogAttrs(globalCtx, slog.LevelInfo, "cannot scale worker thread, max threads reached for worker", slog.String("worker", fc.worker.qualifiedName)) } continue diff --git a/scaling_test.go b/scaling_test.go index d2784a8eeb..a702ea8241 100644 --- a/scaling_test.go +++ b/scaling_test.go @@ -48,7 +48,7 @@ func TestScaleAWorkerThreadUpAndDown(t *testing.T) { autoScaledThread := phpThreads[2] // scale up - scaleWorkerThread(globalWorkersByPath[workerPath], mainThread.done, mainThread.state) + scaleWorkerThread(fallbackServer.workersByPath[workerPath], mainThread.done, mainThread.state) assert.Equal(t, state.Ready, autoScaledThread.state.Get()) // on down-scale, the thread will be marked as inactive diff --git a/server.go b/server.go index 8274f23c54..332eeafa5f 100644 --- a/server.go +++ b/server.go @@ -20,7 +20,7 @@ type Server struct { root string splitPath []string env PreparedEnv - workers []*worker + workersByName map[string]*worker workersByPath map[string]*worker workersWithRequestMatcher []*worker @@ -35,13 +35,15 @@ var ( fallbackServer = newFallbackServer() ) +// newFallbackServer creates the server of requests and workers that are not +// scoped to one, so a lookup is always a lookup in a server func newFallbackServer() *Server { s := &Server{ - idx: -1, - workersByPath: make(map[string]*worker), - env: make(map[string]string), - logger: globalLogger, + idx: -1, + env: make(map[string]string), + logger: globalLogger, } + s.resetWorkers() return s } @@ -54,12 +56,36 @@ func registerServers(newServers []*Server) { fallbackServer.logger = globalLogger fallbackServer.resetWorkers() + // several servers may resolve to the same name (e.g. the same host), but + // the name qualifies worker names in metrics and logs, so it must be + // unique: the first server keeps a name, the next ones get a numeric + // suffix that never takes a name another server configured + configured := make(map[string]struct{}, len(servers)) + for _, s := range servers { + if s.configuredName != "" { + configured[s.configuredName] = struct{}{} + } + } + + taken := make(map[string]struct{}, len(servers)) for i, s := range servers { s.idx = i - s.name = s.configuredName - if s.name == "" { - s.name = "server_" + strconv.Itoa(i) + name := s.configuredName + if name == "" { + name = "server_" + strconv.Itoa(i) } + + for base, n := name, 1; ; n++ { + _, isTaken := taken[name] + _, isConfigured := configured[name] + if !isTaken && (!isConfigured || name == s.configuredName) { + break + } + name = base + "_" + strconv.Itoa(n) + } + taken[name] = struct{}{} + + s.name = name s.resetWorkers() } } @@ -83,7 +109,7 @@ func unregisterServers() { // resetWorkers drops the workers of a previous run; initWorkers() adds them back func (s *Server) resetWorkers() { - s.workers = nil + s.workersByName = make(map[string]*worker) s.workersByPath = make(map[string]*worker) s.workersWithRequestMatcher = nil } @@ -99,9 +125,9 @@ func NewServer(root string, options ...ServerOption) (*Server, error) { } s := &Server{ - root: root, - workersByPath: make(map[string]*worker), + root: root, } + s.resetWorkers() for _, option := range options { if err := option(s); err != nil { @@ -125,20 +151,40 @@ func NewServer(root string, options ...ServerOption) (*Server, error) { } // Name returns the human-readable name of the server. -// It is empty until registration if none was passed to NewServer(). +// It is empty until registration if none was passed to NewServer(), and gets +// a numeric suffix if another registered server has the same name. func (s *Server) Name() string { return s.name } +// addWorker registers a worker scoped to this server +// scope names the worker set in errors: a server, or the global workers +func (s *Server) scope() string { + if s == fallbackServer { + return "two global workers" + } + + return "two workers in a server" +} + func (s *Server) addWorker(w *worker) error { - s.workers = append(s.workers, w) + if s.workersByName[w.name] != nil { + return fmt.Errorf("%s cannot have the same name: %q", s.scope(), w.name) + } + s.workersByName[w.name] = w + + // background workers never serve requests, so they are not matched at all + if w.isBackgroundWorker { + return nil + } + if w.matchRequest != nil { s.workersWithRequestMatcher = append(s.workersWithRequestMatcher, w) return nil } - if _, exists := s.workersByPath[w.fileName]; exists { - return fmt.Errorf("two workers in a server cannot have the same filename: %q", w.fileName) + if s.workersByPath[w.fileName] != nil { + return fmt.Errorf("%s cannot have the same filename: %q", s.scope(), w.fileName) } s.workersByPath[w.fileName] = w diff --git a/server_test.go b/server_test.go index f297db7c29..4bd078040c 100644 --- a/server_test.go +++ b/server_test.go @@ -59,12 +59,59 @@ func TestServer(t *testing.T) { t.Run("name", func(t *testing.T) { named, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("api")) unnamed, _ := frankenphp.NewServer(testDataDir) + alsoNamed, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("api")) + explicit, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("api_1")) - initServers(t, frankenphp.WithServer(named), frankenphp.WithServer(unnamed)) + initServers(t, frankenphp.WithServer(named), frankenphp.WithServer(unnamed), frankenphp.WithServer(alsoNamed), frankenphp.WithServer(explicit)) assert.Equal(t, "api", named.Name()) // an empty name defaults to the server index at registration assert.Equal(t, "server_1", unnamed.Name()) + // names qualify worker names in metrics, so they are made unique, and + // a generated suffix never takes a name another server configured + assert.Equal(t, "api_2", alsoNamed.Name()) + assert.Equal(t, "api_1", explicit.Name()) + }) + + t.Run("same_worker_name_in_two_servers", func(t *testing.T) { + server1, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("one")) + server2, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("two")) + initServers( + t, + frankenphp.WithServer(server1), + frankenphp.WithServer(server2), + frankenphp.WithWorkers("counter", testDataDir+"worker-with-counter.php", 1, frankenphp.WithWorkerServerScope(server1)), + frankenphp.WithWorkers("counter", testDataDir+"worker-with-counter.php", 1, frankenphp.WithWorkerServerScope(server2)), + ) + + // WithWorkerName resolves the name within the request's server + byName := func(server *frankenphp.Server) string { + t.Helper() + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "http://example.com/index.php", nil) + require.NoError(t, server.ServeHTTP(w, req, frankenphp.WithWorkerName("counter"))) + body, err := io.ReadAll(w.Result().Body) + require.NoError(t, err) + + return string(body) + } + + assert.Equal(t, "requests:1", byName(server1)) + assert.Equal(t, "requests:1", byName(server2), "server 2 must get its own worker, not server 1's") + assert.Equal(t, "requests:2", byName(server1)) + }) + + t.Run("error_on_duplicate_worker_names", func(t *testing.T) { + t.Cleanup(frankenphp.Shutdown) + + server, _ := frankenphp.NewServer(testDataDir) + err := frankenphp.Init( + frankenphp.WithServer(server), + frankenphp.WithWorkers("same", testDataDir+"worker-with-counter.php", 1, frankenphp.WithWorkerServerScope(server)), + frankenphp.WithWorkers("same", testDataDir+"index.php", 1, frankenphp.WithWorkerServerScope(server)), + ) + + assert.ErrorContains(t, err, "two workers in a server cannot have the same name") }) t.Run("root", func(t *testing.T) { diff --git a/testdata/_executor.php b/testdata/_executor.php index 61a5319f11..31b87c79cb 100644 --- a/testdata/_executor.php +++ b/testdata/_executor.php @@ -1,7 +1,7 @@ new stdClass()]); + $result = 'no exception'; +} catch (\Throwable $e) { + $result = get_class($e) . ': ' . $e->getMessage(); +} +file_put_contents($_SERVER['BG_SENTINEL'], $result); +$handle = frankenphp_get_worker_handle(); +while (frankenphp_worker_tick()) { + $read = [$handle]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/basic.php b/testdata/bgworker/basic.php new file mode 100644 index 0000000000..9565cef551 --- /dev/null +++ b/testdata/bgworker/basic.php @@ -0,0 +1,20 @@ +getMessage(); +} +file_put_contents($_SERVER['BG_SENTINEL'], $result); +$handle = frankenphp_get_worker_handle(); +while (frankenphp_worker_tick()) { + $read = [$handle]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/count.php b/testdata/bgworker/count.php new file mode 100644 index 0000000000..9c654699ba --- /dev/null +++ b/testdata/bgworker/count.php @@ -0,0 +1,14 @@ + $_SERVER['FRANKENPHP_WORKER'] ?? 'unset', + 'background' => $_SERVER['FRANKENPHP_WORKER_BACKGROUND'] ?? 'unset', +], true)); +$handle = frankenphp_get_worker_handle(); +while (frankenphp_worker_tick()) { + $read = [$handle]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/loop.php b/testdata/bgworker/loop.php new file mode 100644 index 0000000000..f7541eb7dd --- /dev/null +++ b/testdata/bgworker/loop.php @@ -0,0 +1,15 @@ + 1 threads share the name; each touches a file of +// its own under BG_SENTINEL_DIR, then parks on its own handle. +set_time_limit(0); +@touch($_SERVER['BG_SENTINEL_DIR'] . DIRECTORY_SEPARATOR . bin2hex(random_bytes(8))); +$handle = frankenphp_get_worker_handle(); +while (frankenphp_worker_tick()) { + $read = [$handle]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/publisher.php b/testdata/bgworker/publisher.php new file mode 100644 index 0000000000..a395efc801 --- /dev/null +++ b/testdata/bgworker/publisher.php @@ -0,0 +1,20 @@ + 42, + 'value' => $_SERVER['BG_PUBLISH_VALUE'] ?? 'default', + 'nested' => ['a' => 1, 'list' => [true, null, 1.5, 'x']], + 'worker' => $_SERVER['FRANKENPHP_WORKER_BACKGROUND'], +]); +$handle = frankenphp_get_worker_handle(); +while (frankenphp_worker_tick()) { + $read = [$handle]; + $write = $except = null; + stream_select($read, $write, $except, null); +} diff --git a/testdata/bgworker/read.php b/testdata/bgworker/read.php new file mode 100644 index 0000000000..3ef3a66f4c --- /dev/null +++ b/testdata/bgworker/read.php @@ -0,0 +1,10 @@ + 'relayed']); + $result = json_encode(frankenphp_read_task($task)); +} catch (\Throwable $e) { + $result = get_class($e) . ': ' . $e->getMessage(); +} +file_put_contents($_SERVER['BG_SENTINEL'], $result); +frankenphp_worker_tick(); +fgets(frankenphp_get_worker_handle()); diff --git a/testdata/bgworker/task-worker.php b/testdata/bgworker/task-worker.php new file mode 100644 index 0000000000..f3ab73925f --- /dev/null +++ b/testdata/bgworker/task-worker.php @@ -0,0 +1,54 @@ + 0 && feof($stream)) { + throw new \RuntimeException('the sender closed the task before the update'); + } + } + for ($i = 1, $steps = $payload['steps'] ?? 0; $i <= $steps; ++$i) { + frankenphp_update_task($stream, ['step' => $i, 'of' => $steps]); + } + frankenphp_update_task($stream, [ + 'result' => 'processed:' . ($payload['input'] ?? ''), + 'worker' => $_SERVER['FRANKENPHP_WORKER_BACKGROUND'], + 'tag' => $_SERVER['BG_TAG'] ?? '', + 'thread' => $threadId ??= bin2hex(random_bytes(4)), + ]); + } catch (\Throwable $e) { + if (!empty($_SERVER['BG_SENTINEL'])) { + file_put_contents($_SERVER['BG_SENTINEL'], get_class($e) . ': ' . $e->getMessage()); + } + } finally { + fclose($stream); + } + if (!$drain) { + break; + } + } +} diff --git a/testdata/bgworker/tick.php b/testdata/bgworker/tick.php new file mode 100644 index 0000000000..5614f16458 --- /dev/null +++ b/testdata/bgworker/tick.php @@ -0,0 +1,17 @@ +getMessage(), "\n"; + } +} diff --git a/testdata/persist-roundtrip.php b/testdata/persist-roundtrip.php index f2bcec0627..25e0c9dd3a 100644 --- a/testdata/persist-roundtrip.php +++ b/testdata/persist-roundtrip.php @@ -86,3 +86,17 @@ function same(mixed $actual, mixed $expected, string $label): void { } catch (\LogicException) { echo "OK nested stdClass rejected\n"; } + +// A literal array is opcache-immutable and exposed zero-copy: opcache keeps +// its refcount at 2, so exposing it through a refcounted zval would destroy +// shared memory on the second release. Round-trip the same literal more +// times than that. +for ($i = 0; $i < 3; ++$i) { + $out = $rt(['immutable' => ['nested' => [1, 2, 3]], 'i' => 'literal']); + if ($out !== ['immutable' => ['nested' => [1, 2, 3]], 'i' => 'literal']) { + echo "FAIL immutable literal exposed repeatedly (round $i)\n"; + return; + } + unset($out); +} +echo "OK immutable literal exposed repeatedly\n"; diff --git a/testdata/set-vars-outside.php b/testdata/set-vars-outside.php new file mode 100644 index 0000000000..55b689c94d --- /dev/null +++ b/testdata/set-vars-outside.php @@ -0,0 +1,8 @@ + 1]); + echo 'no exception'; +} catch (\Throwable $e) { + echo get_class($e) . ': ' . $e->getMessage(); +} diff --git a/testdata/symlinks/test/index.php b/testdata/symlinks/test/index.php index 15aa1a9cf1..9037dbe762 100644 --- a/testdata/symlinks/test/index.php +++ b/testdata/symlinks/test/index.php @@ -1,7 +1,7 @@ 'slow', 'sleep_ms' => 500]); + try { + frankenphp_send_task('echo', ['input' => 'late'], 0.1); + echo "no timeout\n"; + } catch (\RuntimeException $e) { + echo $e->getMessage(), "\n"; + } + echo json_encode(frankenphp_read_task($slow)); +} catch (\Throwable $e) { + echo get_class($e), ': ', $e->getMessage(); +} diff --git a/testdata/task-errors.php b/testdata/task-errors.php new file mode 100644 index 0000000000..ceb7d6f082 --- /dev/null +++ b/testdata/task-errors.php @@ -0,0 +1,19 @@ + fn () => frankenphp_send_task('nope', []), + 'payload' => fn () => frankenphp_send_task('echo', ['object' => new stdClass()]), + 'timeout' => fn () => frankenphp_send_task('echo', [], -1), + 'receive' => fn () => frankenphp_receive_task(), + 'update' => fn () => frankenphp_update_task(fopen('php://memory', 'r'), []), + 'read' => fn () => frankenphp_read_task(fopen('php://memory', 'r')), +]; +foreach ($cases as $name => $case) { + try { + $case(); + echo $name, ": no exception\n"; + } catch (\Throwable $e) { + echo $name, ': ', get_class($e), ': ', $e->getMessage(), "\n"; + } +} diff --git a/testdata/task-pool.php b/testdata/task-pool.php new file mode 100644 index 0000000000..794bf380b1 --- /dev/null +++ b/testdata/task-pool.php @@ -0,0 +1,30 @@ + 'a', 'sleep_ms' => 300]), + frankenphp_send_task('pool', ['input' => 'b', 'sleep_ms' => 300]), + ]; + $threads = []; + while ($tasks) { + $read = $tasks; + $write = $except = null; + if (!stream_select($read, $write, $except, 5)) { + throw new \RuntimeException('stream_select() timed out'); + } + foreach ($read as $i => $stream) { + if (null === $update = frankenphp_read_task($stream)) { + fclose($stream); + unset($tasks[$i]); + continue; + } + $threads[$update['result']] = $update['thread']; + } + } + ksort($threads); + echo json_encode(array_keys($threads)), "\n", 2 === count(array_unique($threads)) ? 'two threads' : 'one thread'; +} catch (\Throwable $e) { + echo get_class($e), ': ', $e->getMessage(); +} diff --git a/testdata/task-shutdown.php b/testdata/task-shutdown.php new file mode 100644 index 0000000000..cbb9c6aa3b --- /dev/null +++ b/testdata/task-shutdown.php @@ -0,0 +1,13 @@ + 'slow', 'sleep_ms' => 1500, 'mark' => $_GET['mark']]); + frankenphp_send_task('echo', ['input' => 'never'], null); + echo 'picked up'; +} catch (\Throwable $e) { + echo get_class($e), ': ', $e->getMessage(); +} diff --git a/testdata/task.php b/testdata/task.php new file mode 100644 index 0000000000..2089ccdefd --- /dev/null +++ b/testdata/task.php @@ -0,0 +1,23 @@ + is_numeric($v) ? (int) $v : $v, $payload); + $task = frankenphp_send_task($_GET['name'] ?? 'echo', $payload, isset($_GET['timeout']) ? (float) $_GET['timeout'] : 30.0); + if (isset($_GET['close_early'])) { + fclose($task); + echo 'closed'; + + return; + } + while (null !== $update = frankenphp_read_task($task)) { + echo json_encode($update), "\n"; + } + echo 'done'; +} catch (\Throwable $e) { + echo get_class($e), ': ', $e->getMessage(); +} diff --git a/testdata/vars.php b/testdata/vars.php new file mode 100644 index 0000000000..0555109953 --- /dev/null +++ b/testdata/vars.php @@ -0,0 +1,7 @@ +getMessage(); +} diff --git a/testdata/worker-name.php b/testdata/worker-name.php new file mode 100644 index 0000000000..b9f15a047d --- /dev/null +++ b/testdata/worker-name.php @@ -0,0 +1,11 @@ += 0 { + // senders that took the socket before it was withdrawn finish their write first + for handler.signaling.Load() > 0 { + runtime.Gosched() + } + C.frankenphp_worker_close_stop_sock(C.intptr_t(s)) + } +} + +func (handler *backgroundWorkerThread) beforeScriptExecution() string { + return handler.workerLifecycle.beforeScriptExecution(handler.startScript) +} + +// startScript keeps trying to start the script: unlike an HTTP worker, whose +// setup cannot fail, a background worker needs a socket pair per run +func (handler *backgroundWorkerThread) startScript() string { + for { + err := handler.setupScript() + if err == nil { + return handler.worker.fileName + } + + if globalLogger.Enabled(globalCtx, slog.LevelError) { + globalLogger.LogAttrs(globalCtx, slog.LevelError, "failed to start background worker", slog.String("worker", handler.worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex), slog.Any("error", err)) + } + + // fail fast during startup so Init() surfaces the error to the + // operator; past startup, back off and retry like a crash + if reportStartupFailure(err) { + handler.thread.state.Set(state.ShuttingDown) + + return handler.beforeScriptExecution() + } + + handler.backoff() + if !handler.state.Is(state.Ready) && !handler.state.Is(state.TransitionComplete) { + // drained during the backoff (shutdown, reboot, transition) + return handler.beforeScriptExecution() + } + } +} + +// setupScript marks the thread as a background worker on the C side and +// takes ownership of the Go side's end of its stop socket pair. +func (handler *backgroundWorkerThread) setupScript() error { + s := int64(C.frankenphp_set_background_worker_and_get_stop_sock()) + if s < 0 { + return fmt.Errorf("failed to create the stop socket pair of background worker %q", handler.worker.qualifiedName) + } + // tasks queued meanwhile reach the new run when it parks, see + // go_frankenphp_background_worker_park + q := &handler.worker.tasks + q.mu.Lock() + handler.stopSock = s + q.mu.Unlock() + + switch handler.state.Get() { + case state.ShuttingDown, state.Rebooting, state.ForceRebooting, state.TransitionRequested: + // a concurrent drain may have run before the socket was published; + // close it now so the script observes EOF immediately + handler.drain() + } + + fc, err := newWorkerDummyContext(handler.worker) + if err != nil { + handler.drain() + return err + } + handler.dummyFrankenPHPContext = fc + + handler.isBootingScript = true + handler.openTasks = 0 + metrics.StartWorker(handler.worker.qualifiedName) + // the run's logger and context, not the globals: Stop() does not wait + // for a callback that already started, and a shutdown finishing + // meanwhile resets those + logger, ctx, name, threadIndex := fc.logger, fc.ctx, handler.worker.qualifiedName, handler.thread.threadIndex + handler.bootTimer = time.AfterFunc(backgroundBootWarnDelay, func() { + if logger.Enabled(ctx, slog.LevelWarn) { + logger.LogAttrs(ctx, slog.LevelWarn, "background worker has not called frankenphp_worker_tick() yet, Init() and Shutdown() wait for it", slog.String("worker", name), slog.Int("thread", threadIndex)) + } + }) + + if fc.logger.Enabled(fc.ctx, slog.LevelDebug) { + fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "starting background worker", slog.String("worker", handler.worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex)) + } + + // the thread stays in TransitionComplete until the script calls + // frankenphp_worker_tick(), see go_frankenphp_background_worker_ready + + return nil +} + +func (handler *backgroundWorkerThread) afterScriptExecution(exitStatus int) { + // the Go side's end of the stop socket pair belongs to this thread; + // release it on every exit path so the next run gets a fresh pair + // (drain() already took it when the exit was drain-triggered) + handler.drain() + worker := handler.worker + handler.dummyFrankenPHPContext = nil + + handler.stopBootTimer() + handler.state.MarkAsWaiting(false) + + // cooperative exit: the script ticked and returned cleanly, re-run it, + // unless the thread is being drained (beforeScriptExecution checks the + // state) + if exitStatus == 0 && !handler.isBootingScript { + handler.crashCount = 0 + metrics.StopWorker(worker.qualifiedName, StopReasonRestart) + + if globalLogger.Enabled(globalCtx, slog.LevelDebug) { + globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "restarting background worker", slog.String("worker", worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex)) + } + + return + } + + // crash after the ready point: restart without counting toward + // max_consecutive_failures, that cap is about a script that never boots. + // The wait still applies: unlike an HTTP worker, which can only crash + // after a request reached frankenphp_handle_request() and is therefore + // paced by traffic, a background worker reaches its ready point on its + // own and a script crashing right after it would spin + if !handler.isBootingScript { + metrics.StopWorker(worker.qualifiedName, StopReasonCrash) + + if globalLogger.Enabled(globalCtx, slog.LevelWarn) { + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "background worker crashed, restarting", slog.String("worker", worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus), slog.Int("crashes", handler.crashCount)) + } + + time.Sleep(restartBackoff(handler.crashCount)) + handler.crashCount++ + + return + } + + // boot failure: the script exited before calling frankenphp_worker_tick(), + // a clean exit included, which would otherwise respawn in a tight loop. + // StopReasonBootFailure skips the ready-gauge decrement, matching the + // ReadyWorker call that never happened + metrics.StopWorker(worker.qualifiedName, StopReasonBootFailure) + + // max_consecutive_failures only fails hard during startup, where it + // surfaces on startupFailChan so Init() returns the error to the + // operator. Past startup, a failing background worker keeps + // restarting with a louder log line: silently giving up would leave + // the server in a broken half-state with no clear way to recover. + pastCap := worker.maxConsecutiveFailures >= 0 && handler.failureCount >= worker.maxConsecutiveFailures + if pastCap && !watcherIsEnabled { + var err error + if exitStatus == 0 { + err = fmt.Errorf("background worker %s exits without calling frankenphp_worker_tick()", worker.fileName) + } else { + err = fmt.Errorf("too many consecutive failures: background worker %s keeps crashing", worker.fileName) + } + if reportStartupFailure(err) { + handler.thread.state.Set(state.ShuttingDown) + return + } + } + + logLevel := slog.LevelWarn + logMsg := "background worker failed before calling frankenphp_worker_tick(), restarting" + if exitStatus == 0 { + logMsg = "background worker exited without calling frankenphp_worker_tick(), restarting" + } + if pastCap { + logLevel = slog.LevelError + logMsg = "background worker exceeded max_consecutive_failures, still restarting" + } + if globalLogger.Enabled(globalCtx, logLevel) { + globalLogger.LogAttrs(globalCtx, logLevel, logMsg, slog.String("worker", worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex), slog.Int("failures", handler.failureCount), slog.Int("exit_status", exitStatus)) + } + + handler.backoff() +} + +func (handler *backgroundWorkerThread) stopBootTimer() { + if handler.bootTimer != nil { + handler.bootTimer.Stop() + handler.bootTimer = nil + } +} + +//export go_frankenphp_background_worker_ready +func go_frankenphp_background_worker_ready(threadIndex C.uintptr_t) { + // called on the PHP thread by the first frankenphp_worker_tick() of a + // run; the handler is a backgroundWorkerThread because that function + // throws on every other thread kind + if handler, ok := phpThreads[threadIndex].handler.(*backgroundWorkerThread); ok && handler.isBootingScript { + handler.isBootingScript = false + // the boot succeeded, only consecutive boot failures count + handler.failureCount = 0 + handler.stopBootTimer() + handler.worker.markReady() + metrics.ReadyWorker(handler.worker.qualifiedName) + // parked from now on as far as the threads state endpoint is concerned + handler.state.MarkAsWaiting(true) + + // like an HTTP worker reaching frankenphp_handle_request(), the thread + // is ready only now: initWorkers() waits for this state, so a script + // that fails before its first tick still fails Init() + if handler.state.Is(state.TransitionComplete) { + handler.state.Set(state.Ready) + } + } +} + +// backoff waits before the next run of a crashed script, see restartBackoff +func (handler *backgroundWorkerThread) backoff() { + time.Sleep(restartBackoff(handler.failureCount)) + handler.failureCount++ +} diff --git a/threadworker.go b/threadworker.go index 8a308b4d87..c37e1fb674 100644 --- a/threadworker.go +++ b/threadworker.go @@ -15,9 +15,8 @@ import ( // executes the PHP worker script in a loop // implements the threadHandler interface type workerThread struct { - state *state.ThreadState - thread *phpThread - worker *worker + workerLifecycle + dummyFrankenPHPContext *frankenPHPContext workerFrankenPHPContext *frankenPHPContext isBootingScript bool // true if the worker has not reached frankenphp_handle_request yet @@ -26,49 +25,19 @@ type workerThread struct { } func convertToWorkerThread(thread *phpThread, worker *worker) { - thread.setHandler(&workerThread{ - state: thread.state, - thread: thread, - worker: worker, - }) + thread.setHandler(&workerThread{workerLifecycle: newWorkerLifecycle(thread, worker)}) worker.attachThread(thread) } -// beforeScriptExecution returns the name of the script or an empty string on shutdown func (handler *workerThread) beforeScriptExecution() string { - switch handler.state.Get() { - case state.TransitionRequested: - if handler.worker.onThreadShutdown != nil { - handler.worker.onThreadShutdown(handler.thread.threadIndex) - } - handler.worker.detachThread(handler.thread) - return handler.thread.transitionToNewHandler() - case state.Ready, state.TransitionComplete: - handler.thread.updateContext(true) - if handler.worker.onThreadReady != nil { - handler.worker.onThreadReady(handler.thread.threadIndex) - } - - setupWorkerScript(handler, handler.worker) + return handler.workerLifecycle.beforeScriptExecution(handler.startScript) +} - return handler.worker.fileName - case state.Rebooting, state.ForceRebooting: - return "" - case state.RebootReady: - handler.requestCount = 0 - handler.state.Set(state.Ready) - return handler.beforeScriptExecution() - case state.ShuttingDown: - if handler.worker.onThreadShutdown != nil { - handler.worker.onThreadShutdown(handler.thread.threadIndex) - } - handler.worker.detachThread(handler.thread) +// startScript runs the worker script; it always has one to run +func (handler *workerThread) startScript() string { + setupWorkerScript(handler, handler.worker) - // signal to stop - return "" - default: - panic("unexpected state: " + handler.state.Name()) - } + return handler.worker.fileName } func (handler *workerThread) afterScriptExecution(exitStatus int) { @@ -90,7 +59,7 @@ func (handler *workerThread) name() string { func (handler *workerThread) drain() {} func setupWorkerScript(handler *workerThread, worker *worker) { - metrics.StartWorker(worker.name) + metrics.StartWorker(worker.qualifiedName) // Create a dummy request to set up the worker fc, err := newWorkerDummyContext(worker) @@ -103,7 +72,7 @@ func setupWorkerScript(handler *workerThread, worker *worker) { handler.requestCount = 0 if fc.logger.Enabled(fc.ctx, slog.LevelDebug) { - fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "starting", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex)) + fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "starting", slog.String("worker", worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex)) } } @@ -122,10 +91,10 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { // on exit status 0 we just run the worker script again if exitStatus == 0 && !handler.isBootingScript { - metrics.StopWorker(worker.name, StopReasonRestart) + metrics.StopWorker(worker.qualifiedName, StopReasonRestart) if globalLogger.Enabled(globalCtx, slog.LevelDebug) { - globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "restarting", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus)) + globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "restarting", slog.String("worker", worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus)) } return @@ -133,9 +102,9 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { // worker has thrown a fatal error or has not reached frankenphp_handle_request if handler.isBootingScript { - metrics.StopWorker(worker.name, StopReasonBootFailure) + metrics.StopWorker(worker.qualifiedName, StopReasonBootFailure) } else { - metrics.StopWorker(worker.name, StopReasonCrash) + metrics.StopWorker(worker.qualifiedName, StopReasonCrash) } if !handler.isBootingScript { @@ -143,14 +112,14 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { // unlike a clean restart, this took down any in-flight request, so // surface it above debug level, with the exit status needed to triage it if globalLogger.Enabled(globalCtx, slog.LevelWarn) { - globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "unexpected termination, restarting", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus)) + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "unexpected termination, restarting", slog.String("worker", worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex), slog.Int("exit_status", exitStatus)) } return } - if worker.maxConsecutiveFailures >= 0 && startupFailChan != nil && !watcherIsEnabled && handler.failureCount >= worker.maxConsecutiveFailures { - startupFailChan <- fmt.Errorf("too many consecutive failures: worker %s has not reached frankenphp_handle_request()", worker.fileName) + if worker.maxConsecutiveFailures >= 0 && !watcherIsEnabled && handler.failureCount >= worker.maxConsecutiveFailures && + reportStartupFailure(fmt.Errorf("too many consecutive failures: worker %s has not reached frankenphp_handle_request()", worker.fileName)) { handler.thread.state.Set(state.ShuttingDown) return } @@ -158,23 +127,32 @@ func tearDownWorkerScript(handler *workerThread, exitStatus int) { if watcherIsEnabled { // worker script has probably failed due to script changes while watcher is enabled if globalLogger.Enabled(globalCtx, slog.LevelError) { - globalLogger.LogAttrs(globalCtx, slog.LevelError, "(watcher enabled) worker script has not reached frankenphp_handle_request()", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex)) + globalLogger.LogAttrs(globalCtx, slog.LevelError, "(watcher enabled) worker script has not reached frankenphp_handle_request()", slog.String("worker", worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex)) } } else { // rare case where worker script has failed on a restart during normal operation // this can happen if startup success depends on external resources if globalLogger.Enabled(globalCtx, slog.LevelWarn) { - globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "worker script has failed on restart", slog.String("worker", worker.name), slog.Int("thread", handler.thread.threadIndex), slog.Int("failures", handler.failureCount)) + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "worker script has failed on restart", slog.String("worker", worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex), slog.Int("failures", handler.failureCount)) } } - // wait a bit and try again (exponential backoff) - backoffDuration := time.Duration(handler.failureCount*handler.failureCount*100) * time.Millisecond - if backoffDuration > time.Second { - backoffDuration = time.Second - } + // wait a bit and try again + time.Sleep(restartBackoff(handler.failureCount)) handler.failureCount++ - time.Sleep(backoffDuration) +} + +// restartBackoff is the wait before a worker script is re-run after a +// failure: quadratic in the number of consecutive failures, capped at one +// second; shared by HTTP and background workers. The cap comes before the +// multiplication, which overflows a duration past some 300k failures, a +// count a crash loop reaches on its own in a few days +func restartBackoff(failures int) time.Duration { + if failures >= 4 { + return time.Second + } + + return time.Duration(failures*failures*100) * time.Millisecond } // waitForWorkerRequest is called during frankenphp_handle_request in the php worker script. @@ -183,7 +161,7 @@ func (handler *workerThread) waitForWorkerRequest() (bool, any) { handler.thread.Unpin() if globalLogger.Enabled(globalCtx, slog.LevelDebug) { - globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "waiting for request", slog.String("worker", handler.worker.name), slog.Int("thread", handler.thread.threadIndex)) + globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "waiting for request", slog.String("worker", handler.worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex)) } // Clear the first dummy request created to initialize the worker @@ -195,14 +173,14 @@ func (handler *workerThread) waitForWorkerRequest() (bool, any) { } // worker is truly ready only after reaching frankenphp_handle_request() - metrics.ReadyWorker(handler.worker.name) + metrics.ReadyWorker(handler.worker.qualifiedName) } // max_requests reached: signal reboot for full ZTS cleanup if maxRequestsPerThread > 0 && handler.requestCount >= maxRequestsPerThread { if globalLogger.Enabled(globalCtx, slog.LevelDebug) { globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "max requests reached, restarting", - slog.String("worker", handler.worker.name), + slog.String("worker", handler.worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex), slog.Int("max_requests", maxRequestsPerThread), ) @@ -223,7 +201,7 @@ func (handler *workerThread) waitForWorkerRequest() (bool, any) { select { case <-handler.thread.drainChan: if globalLogger.Enabled(globalCtx, slog.LevelDebug) { - globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "shutting down", slog.String("worker", handler.worker.name), slog.Int("thread", handler.thread.threadIndex)) + globalLogger.LogAttrs(globalCtx, slog.LevelDebug, "shutting down", slog.String("worker", handler.worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex)) } return false, nil @@ -239,9 +217,9 @@ func (handler *workerThread) waitForWorkerRequest() (bool, any) { if fc.logger.Enabled(fc.ctx, slog.LevelDebug) { if handler.workerFrankenPHPContext.request == nil { - fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "request handling started", slog.String("worker", handler.worker.name), slog.Int("thread", handler.thread.threadIndex)) + fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "request handling started", slog.String("worker", handler.worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex)) } else { - fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "request handling started", slog.String("worker", handler.worker.name), slog.Int("thread", handler.thread.threadIndex), slog.String("url", handler.workerFrankenPHPContext.request.RequestURI)) + fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "request handling started", slog.String("worker", handler.worker.qualifiedName), slog.Int("thread", handler.thread.threadIndex), slog.String("url", handler.workerFrankenPHPContext.request.RequestURI)) } } @@ -298,9 +276,9 @@ func go_frankenphp_finish_worker_request(threadIndex C.uintptr_t, retval *C.zval if fc.logger.Enabled(fc.ctx, slog.LevelDebug) { if fc.request == nil { - fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "request handling finished", slog.String("worker", fc.worker.name), slog.Int("thread", thread.threadIndex)) + fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "request handling finished", slog.String("worker", fc.worker.qualifiedName), slog.Int("thread", thread.threadIndex)) } else { - fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "request handling finished", slog.String("worker", fc.worker.name), slog.Int("thread", thread.threadIndex), slog.String("url", fc.request.RequestURI)) + fc.logger.LogAttrs(fc.ctx, slog.LevelDebug, "request handling finished", slog.String("worker", fc.worker.qualifiedName), slog.Int("thread", thread.threadIndex), slog.String("url", fc.request.RequestURI)) } } } diff --git a/threadworker_test.go b/threadworker_test.go new file mode 100644 index 0000000000..24e5a2e410 --- /dev/null +++ b/threadworker_test.go @@ -0,0 +1,20 @@ +package frankenphp + +import ( + "math" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestRestartBackoff(t *testing.T) { + assert.Equal(t, time.Duration(0), restartBackoff(0)) + assert.Equal(t, 100*time.Millisecond, restartBackoff(1)) + assert.Equal(t, 400*time.Millisecond, restartBackoff(2)) + assert.Equal(t, 900*time.Millisecond, restartBackoff(3)) + assert.Equal(t, time.Second, restartBackoff(4)) + // a crash loop counts without bound, the quadratic must not overflow + assert.Equal(t, time.Second, restartBackoff(303701)) + assert.Equal(t, time.Second, restartBackoff(math.MaxInt32)) +} diff --git a/types_test.go b/types_test.go index a08f90725e..7d547e9681 100644 --- a/types_test.go +++ b/types_test.go @@ -14,7 +14,7 @@ func testOnDummyPHPThread(t *testing.T, test func()) { t.Helper() globalLogger = slog.Default() - _, err := initPHPThreads(1, 1, nil) // boot 1 thread + _, err := initPHPThreads(1, 1, 0, nil) // boot 1 thread assert.NoError(t, err) handler := convertToTaskThread(phpThreads[0]) diff --git a/worker.go b/worker.go index 388dfbd031..51b101c12e 100644 --- a/worker.go +++ b/worker.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "runtime" + "slices" "sync" "sync/atomic" "time" @@ -20,7 +21,11 @@ import ( type worker struct { mercureContext - name string + // name as declared, unique within its server (or among global workers) + name string + // qualifiedName is unique across the process: ":" for + // server-scoped workers, name otherwise; used for metrics and logs + qualifiedName string fileName string matchRequest func(*http.Request) bool num int @@ -34,14 +39,32 @@ type worker struct { onThreadShutdown func(int) queuedRequests atomic.Int32 server *Server + // isBackgroundWorker marks this as a background (non-HTTP) worker + isBackgroundWorker bool + // readyOnce is closed the first time a thread of a background worker + // reaches its ready point; frankenphp_get_vars() readers wait on it + readyOnce chan struct{} + readyClose sync.Once + // vars is the snapshot published with frankenphp_set_vars() + vars varsSlot + // tasks holds the tasks sent with frankenphp_send_task() until a thread + // picks them up + tasks taskQueue +} + +// markReady records that the background worker reached its ready point once +func (worker *worker) markReady() { + worker.readyClose.Do(func() { close(worker.readyOnce) }) } var ( - workers []*worker - workersByName map[string]*worker - globalWorkersByPath map[string]*worker - watcherIsEnabled bool - startupFailChan chan error + workers []*worker + watcherIsEnabled bool + startupFailChan chan error + // startupPhase is true while initWorkers() waits for the workers to + // boot, the only time a boot failure must reach startupFailChan: past + // that point the handlers log and keep restarting on their own + startupPhase atomic.Bool ) func initWorkers(opts []workerOpt) error { @@ -55,8 +78,7 @@ func initWorkers(opts []workerOpt) error { ) workers = make([]*worker, 0, len(opts)) - workersByName = make(map[string]*worker, len(opts)) - globalWorkersByPath = make(map[string]*worker, len(opts)) + qualifiedNames := make(map[string]bool, len(opts)) for _, o := range opts { w, err := newWorker(o) @@ -64,22 +86,40 @@ func initWorkers(opts []workerOpt) error { return err } - totalThreadsToStart += w.num - workers = append(workers, w) - workersByName[w.name] = w - if w.server == nil { - globalWorkersByPath[w.fileName] = w - } else if err := w.server.addWorker(w); err != nil { + if w.server != fallbackServer && !slices.Contains(servers, w.server) { + return fmt.Errorf("worker %q is scoped to a server that was not passed to WithServer()", w.name) + } + + // names and paths are unique within a server + if err := w.server.addWorker(w); err != nil { return err } + + // scoping makes qualified names unique in all but pathological + // cases: a global worker may still be named like the ":" + // of a scoped one, and metrics would merge the two series + if qualifiedNames[w.qualifiedName] { + return fmt.Errorf("two workers cannot report under the same name: %q", w.qualifiedName) + } + qualifiedNames[w.qualifiedName] = true + + totalThreadsToStart += w.num + workers = append(workers, w) + // reported here rather than in calculateMaxThreads(), where the name is not resolved yet + metrics.TotalWorkers(w.qualifiedName, w.num) } startupFailChan = make(chan error, totalThreadsToStart) + startupPhase.Store(true) for _, w := range workers { for range w.num { thread := getInactivePHPThread() - convertToWorkerThread(thread, w) + if w.isBackgroundWorker { + convertToBackgroundWorkerThread(thread, w) + } else { + convertToWorkerThread(thread, w) + } workersReady.Go(func() { thread.state.WaitFor(state.Ready, state.ShuttingDown, state.Done) @@ -88,6 +128,7 @@ func initWorkers(opts []workerOpt) error { } workersReady.Wait() + startupPhase.Store(false) select { case err := <-startupFailChan: @@ -95,12 +136,29 @@ func initWorkers(opts []workerOpt) error { return fmt.Errorf("failed to initialize workers: %w", err) default: // all workers started successfully - startupFailChan = nil } return nil } +// reportStartupFailure hands a boot failure to initWorkers() while it waits +// for the workers, so Init() fails, and reports whether it did: past that +// point the failure is dropped, the handler has logged it and keeps +// restarting. It never blocks, the buffer holds one error per thread and a +// thread failing repeatedly in the startup window must not hang on it +func reportStartupFailure(err error) bool { + if !startupPhase.Load() { + return false + } + + select { + case startupFailChan <- err: + default: + } + + return true +} + func newWorker(o workerOpt) (*worker, error) { // Order is important! // This order ensures that FrankenPHP started from inside a symlinked directory will properly resolve any paths. @@ -119,23 +177,38 @@ func newWorker(o workerOpt) (*worker, error) { return nil, fmt.Errorf("worker file not found %q: %w", absFileName, err) } + if o.isBackgroundWorker { + // the name is the script's identity (exposed via FRANKENPHP_WORKER); + // empty names are reserved for the catch-all workers of a future build + if o.name == "" { + return nil, fmt.Errorf("background worker %q must have an explicit name", o.fileName) + } + if o.matchRequest != nil { + return nil, fmt.Errorf("background worker %q cannot match requests", o.name) + } + if o.maxThreads > 0 { + return nil, fmt.Errorf("background worker %q cannot set max_threads, it does not autoscale", o.name) + } + // Workers.SendRequest() and SendMessage() dispatch on requestChan, + // which no background thread reads + if o.extensionWorkers != nil { + return nil, fmt.Errorf("background worker %q cannot be an extension worker, those handle requests", o.name) + } + } + if o.name == "" { o.name = absFileName } - if o.server == nil { - if globalWorkersByPath[absFileName] != nil { - return nil, fmt.Errorf("two global workers cannot have the same filename: %q", absFileName) - } - - // no server means no set of requests to match against, the matcher would never run - if o.matchRequest != nil { - return nil, fmt.Errorf("worker %q has a request matcher but no server scope, use WithWorkerServerScope()", o.name) - } + // no server means no set of requests to match against, the matcher would never run + if o.server == nil && o.matchRequest != nil { + return nil, fmt.Errorf("worker %q has a request matcher but no server scope, use WithWorkerServerScope()", o.name) } - if workersByName[o.name] != nil { - return nil, fmt.Errorf("two workers cannot have the same name: %q", o.name) + // the same name may be declared in several servers, metrics and logs need a unique one + qualifiedName := o.name + if o.server != nil { + qualifiedName = o.server.name + ":" + o.name } // env should always contain FRANKENPHP_WORKER and the parent php_server env @@ -152,10 +225,19 @@ func newWorker(o workerOpt) (*worker, error) { } } - o.env["FRANKENPHP_WORKER\x00"] = "1" + // $_SERVER['FRANKENPHP_WORKER'] identifies an HTTP worker, as it always + // did, and $_SERVER['FRANKENPHP_WORKER_BACKGROUND'] a background one, + // holding its declared name: a script serving both roles tests which of + // the two is set + if o.isBackgroundWorker { + o.env["FRANKENPHP_WORKER_BACKGROUND\x00"] = o.name + } else { + o.env["FRANKENPHP_WORKER\x00"] = "1" + } w := &worker{ name: o.name, + qualifiedName: qualifiedName, fileName: absFileName, matchRequest: o.matchRequest, requestOptions: o.requestOptions, @@ -167,6 +249,15 @@ func newWorker(o workerOpt) (*worker, error) { onThreadReady: o.onThreadReady, onThreadShutdown: o.onThreadShutdown, server: o.server, + isBackgroundWorker: o.isBackgroundWorker, + readyOnce: make(chan struct{}), + } + + // a worker declared without a scope belongs to the fallback server, the + // one serving the requests that have no server either, so a worker + // always has one + if w.server == nil { + w.server = fallbackServer } w.configureMercure(&o) @@ -235,7 +326,7 @@ func (worker *worker) isAtThreadLimit() bool { } func (worker *worker) handleRequest(fc *frankenPHPContext) error { - metrics.StartWorkerRequest(worker.name) + metrics.StartWorkerRequest(worker.qualifiedName) runtime.Gosched() @@ -247,7 +338,7 @@ func (worker *worker) handleRequest(fc *frankenPHPContext) error { case thread.requestChan <- fc: worker.threadMutex.RUnlock() <-fc.done - metrics.StopWorkerRequest(worker.name, time.Since(fc.startedAt)) + metrics.StopWorkerRequest(worker.qualifiedName, time.Since(fc.startedAt)) return nil default: @@ -259,7 +350,7 @@ func (worker *worker) handleRequest(fc *frankenPHPContext) error { // if no thread was available, mark the request as queued and apply the scaling strategy worker.queuedRequests.Add(1) - metrics.QueuedWorkerRequest(worker.name) + metrics.QueuedWorkerRequest(worker.qualifiedName) for { workerScaleChan := scaleChan @@ -270,9 +361,9 @@ func (worker *worker) handleRequest(fc *frankenPHPContext) error { select { case worker.requestChan <- fc: worker.queuedRequests.Add(-1) - metrics.DequeuedWorkerRequest(worker.name) + metrics.DequeuedWorkerRequest(worker.qualifiedName) <-fc.done - metrics.StopWorkerRequest(worker.name, time.Since(fc.startedAt)) + metrics.StopWorkerRequest(worker.qualifiedName, time.Since(fc.startedAt)) return nil case workerScaleChan <- fc: @@ -280,8 +371,8 @@ func (worker *worker) handleRequest(fc *frankenPHPContext) error { case <-timeoutChan(time.Duration(maxWaitTime.Load())): // the request has timed out stalling worker.queuedRequests.Add(-1) - metrics.DequeuedWorkerRequest(worker.name) - metrics.StopWorkerRequest(worker.name, time.Since(fc.startedAt)) + metrics.DequeuedWorkerRequest(worker.qualifiedName) + metrics.StopWorkerRequest(worker.qualifiedName, time.Since(fc.startedAt)) fc.reject(ErrMaxWaitTimeExceeded) diff --git a/workerextension.go b/workerextension.go index 814eea1565..01dab1d58b 100644 --- a/workerextension.go +++ b/workerextension.go @@ -26,25 +26,30 @@ type extensionWorkers struct { // EXPERIMENTAL: SendRequest sends an HTTP request to the worker and writes the response to the provided ResponseWriter. func (w *extensionWorkers) SendRequest(rw http.ResponseWriter, r *http.Request) error { - fr, err := NewRequestWithContext( - r, - WithOriginalRequest(r), - WithWorkerName(w.name), - ) - - if err != nil { - return err + // the worker only exists between Init() and Shutdown() + if w.internalWorker == nil { + return ErrNotRunning } - return ServeHTTP(rw, fr) + // worker names are resolved within a server, and a worker always has + // one, the fallback server when it was declared without a scope + return w.internalWorker.server.ServeHTTP(rw, r, WithOriginalRequest(r), WithWorkerName(w.name)) } func (w *extensionWorkers) NumThreads() int { + if w.internalWorker == nil { + return 0 + } + return w.internalWorker.countThreads() } // EXPERIMENTAL: SendMessage sends a message to the worker and waits for a response. func (w *extensionWorkers) SendMessage(ctx context.Context, message any, rw http.ResponseWriter) (any, error) { + if w.internalWorker == nil { + return nil, ErrNotRunning + } + fc := newContextFromMessage(message, rw, ctx, w.internalWorker) err := w.internalWorker.handleRequest(fc) diff --git a/workerextension_test.go b/workerextension_test.go index e861ec97a3..f8dfab0a23 100644 --- a/workerextension_test.go +++ b/workerextension_test.go @@ -77,6 +77,36 @@ func TestWorkerExtensionSendMessage(t *testing.T) { assert.Equal(t, "received message: Hello Workers", ret) } +// an extension worker scoped to a server stays reachable through +// SendRequest(), which resolves the name within that server +func TestWorkerExtensionOnServer(t *testing.T) { + t.Cleanup(Shutdown) + + server, err := NewServer("testdata/", WithServerName("api")) + require.NoError(t, err) + externalWorker, o := WithExtensionWorkers("scopedWorker", "testdata/worker.php", 1, WithWorkerServerScope(server)) + require.NoError(t, Init(o, WithServer(server))) + + // a URI that does not match the worker's own script, so only the name + // lookup can route this to it + w := httptest.NewRecorder() + require.NoError(t, externalWorker.SendRequest(w, httptest.NewRequest("GET", "http://example.com/index.php", nil))) + + body, err := io.ReadAll(w.Result().Body) + require.NoError(t, err) + assert.Contains(t, string(body), "Requests handled: 0") +} + +// background workers never read requestChan, so an extension cannot send +// them requests or messages +func TestErrorIfExtensionWorkerIsBackground(t *testing.T) { + t.Cleanup(Shutdown) + + _, o := WithExtensionWorkers("backgroundExtension", "testdata/bgworker/basic.php", 1, WithWorkerBackground()) + + require.ErrorContains(t, Init(o), "cannot be an extension worker") +} + func TestErrorIf2WorkersHaveSameName(t *testing.T) { _, o1 := WithExtensionWorkers("duplicateWorker", "testdata/worker.php", 1) _, o2 := WithExtensionWorkers("duplicateWorker", "testdata/worker2.php", 1) diff --git a/workerlifecycle.go b/workerlifecycle.go new file mode 100644 index 0000000000..4d1809ee22 --- /dev/null +++ b/workerlifecycle.go @@ -0,0 +1,58 @@ +package frankenphp + +import "github.com/dunglas/frankenphp/internal/state" + +// workerLifecycle is the part of a worker thread that HTTP and background +// workers share: the thread, its worker, and the states both walk through +// between two runs of the script. Handlers embed it and pass the one step +// that differs, how a run starts. +type workerLifecycle struct { + state *state.ThreadState + thread *phpThread + worker *worker +} + +func newWorkerLifecycle(thread *phpThread, worker *worker) workerLifecycle { + return workerLifecycle{state: thread.state, thread: thread, worker: worker} +} + +// beforeScriptExecution returns the name of the script to run, or an empty +// string to stop the thread; startScript prepares a run and returns the +// script to execute, or an empty string to stop the thread +func (l *workerLifecycle) beforeScriptExecution(startScript func() string) string { + switch l.state.Get() { + case state.TransitionRequested: + l.detach() + + return l.thread.transitionToNewHandler() + case state.Ready, state.TransitionComplete: + l.thread.updateContext(true) + if l.worker.onThreadReady != nil { + l.worker.onThreadReady(l.thread.threadIndex) + } + + return startScript() + case state.Rebooting, state.ForceRebooting: + return "" + case state.RebootReady: + l.state.Set(state.Ready) + + return l.beforeScriptExecution(startScript) + case state.ShuttingDown: + l.detach() + + // signal to stop + return "" + default: + panic("unexpected state: " + l.state.Name()) + } +} + +// detach takes the thread off its worker, on the paths that stop running its +// script for good +func (l *workerLifecycle) detach() { + if l.worker.onThreadShutdown != nil { + l.worker.onThreadShutdown(l.thread.threadIndex) + } + l.worker.detachThread(l.thread) +} diff --git a/workertask.go b/workertask.go new file mode 100644 index 0000000000..f457bb7cbb --- /dev/null +++ b/workertask.go @@ -0,0 +1,579 @@ +package frankenphp + +// #include "frankenphp.h" +import "C" +import ( + "runtime/cgo" + "slices" + "strconv" + "sync" + "time" +) + +// taskUpdatesMax bounds the updates buffered per task: past it, +// frankenphp_update_task() waits for the sender to read +const taskUpdatesMax = 16 + +// taskSignalEscalation bounds how long a task waits on the one thread it was +// signaled to: past it every thread gets the line, so a script that parked +// its handle without reading it does not hold the task +const taskSignalEscalation = 10 * time.Millisecond + +// workerTask is a unit of work handed by a PHP thread to a thread of a +// background worker, see frankenphp_send_task(). The payload and the +// updates flowing back are persistent HashTables, copied into request +// memory on arrival. Each side waits on its descriptor of the task's channel +// and is signaled there by the other, one signal per event: pickup, update, +// completion and abort for the sender, abandonment for the receiver. +type workerTask struct { + handle cgo.Handle + worker *worker + payload *C.HashTable // owned by the task until a thread picks it up + pickedUp chan struct{} // closed when a thread picks the task up + // cancelled is closed when the sender gave up before any pickup, ending + // the watcher; abortReason is set by the watcher, under the queue mutex, + // when the wait must end without a pickup + cancelled chan struct{} + abortReason string + // receiver and pickedUpAt are set by the thread that picked the task + // up and read by its close, on the same thread + receiver *backgroundWorkerThread + pickedUpAt time.Time + // fds[0] is the sender's descriptor, fds[1] the receiver's; the streams + // wait on them but the task owns them, until both sides closed and the + // pair goes back to the pool + fds [2]int64 + + mu sync.Mutex + cond *sync.Cond // signaled on pop and close + updates []*C.HashTable + closed bool // the receiver closed its stream + aborted bool // ...during request shutdown: the script ended with the task open + senderGone bool // the sender closed its stream + retired int // sides done with the task, freed at 2 +} + +// taskQueue holds the tasks sent to a background worker until a thread picks +// them up. Its mutex also guards the stop sockets of the worker's threads: +// senders write the wake-up line to them, so they must not be closed +// meanwhile. +type taskQueue struct { + mu sync.Mutex + pending []*workerTask + next int // thread to signal first, spreads tasks over a pool +} + +// remove takes t out of the queue; false if a thread picked it up already +func (q *taskQueue) remove(t *workerTask) bool { + q.mu.Lock() + defer q.mu.Unlock() + + i := slices.Index(q.pending, t) + if i < 0 { + return false + } + q.pending = slices.Delete(q.pending, i, i+1) + + return true +} + +// claimParkedThread picks one parked thread of the worker, round-robin over +// the pool, and returns its stop socket to write the wake-up line to, or -1 +// when no thread is parked: the task then waits in the queue for a thread to +// drain it or to park, see go_frankenphp_background_worker_park. The thread +// is no longer parked once claimed. Called with tasks.mu held; the caller +// writes after releasing it, see signalThreads +func (worker *worker) claimParkedThread() (*backgroundWorkerThread, int64) { + worker.threadMutex.RLock() + defer worker.threadMutex.RUnlock() + + n := len(worker.threads) + for i := range n { + thread := worker.threads[(worker.tasks.next+i)%n] + if handler, ok := thread.handler.(*backgroundWorkerThread); ok && handler.parked && handler.stopSock >= 0 { + handler.parked = false + handler.signaling.Add(1) + worker.tasks.next = (worker.tasks.next + i + 1) % n + + return handler, handler.stopSock + } + } + + return nil, -1 +} + +// claimAllThreads is the fallback of taskSignalEscalation: every thread of +// the worker gets the line, parked or not. Called with tasks.mu held, the +// caller writes to the sockets after releasing it +func (worker *worker) claimAllThreads() (handlers []*backgroundWorkerThread, socks []int64) { + worker.threadMutex.RLock() + for _, thread := range worker.threads { + if handler, ok := thread.handler.(*backgroundWorkerThread); ok && handler.stopSock >= 0 { + handler.parked = false + handler.signaling.Add(1) + handlers = append(handlers, handler) + socks = append(socks, handler.stopSock) + } + } + worker.threadMutex.RUnlock() + + return handlers, socks +} + +// signalThreads writes the wake-up line to sockets claimed under tasks.mu, +// after it was released: the write is a syscall, and a thread contending +// for the mutex meanwhile would park at the price of a scheduler hand-off +func signalThreads(handlers []*backgroundWorkerThread, socks []int64) { + for i, s := range socks { + C.frankenphp_worker_signal_task(C.intptr_t(s)) + handlers[i].signaling.Add(-1) + } +} + +// taskChanPool keeps the descriptor pairs of finished tasks for the next +// ones: drained, they are as good as new, and creating and closing them was +// most of a task's syscalls. Bounded so an idle server does not hold the +// descriptors of a past peak. +var taskChanPool struct { + mu sync.Mutex + free [][2]int64 +} + +const taskChanPoolMax = 256 + +// taskChanGet returns a drained pair from the pool, or a new one +func taskChanGet() ([2]int64, bool) { + taskChanPool.mu.Lock() + if n := len(taskChanPool.free); n > 0 { + fds := taskChanPool.free[n-1] + taskChanPool.free = taskChanPool.free[:n-1] + taskChanPool.mu.Unlock() + + return fds, true + } + taskChanPool.mu.Unlock() + + var fds [2]C.intptr_t + if C.frankenphp_task_chan_open(&fds[0]) != 0 { + return [2]int64{}, false + } + + return [2]int64{int64(fds[0]), int64(fds[1])}, true +} + +// taskChanPut returns a pair to the pool, closed if the pool is full; the +// syscalls happen outside of the pool mutex +func taskChanPut(fds [2]int64) { + C.frankenphp_task_chan_drain(C.intptr_t(fds[0])) + C.frankenphp_task_chan_drain(C.intptr_t(fds[1])) + + taskChanPool.mu.Lock() + if len(taskChanPool.free) < taskChanPoolMax { + taskChanPool.free = append(taskChanPool.free, fds) + taskChanPool.mu.Unlock() + + return + } + taskChanPool.mu.Unlock() + + C.frankenphp_close_sock(C.intptr_t(fds[0])) + C.frankenphp_close_sock(C.intptr_t(fds[1])) +} + +// freeTaskChans closes the pooled pairs on shutdown +func freeTaskChans() { + taskChanPool.mu.Lock() + free := taskChanPool.free + taskChanPool.free = nil + taskChanPool.mu.Unlock() + + for _, fds := range free { + C.frankenphp_close_sock(C.intptr_t(fds[0])) + C.frankenphp_close_sock(C.intptr_t(fds[1])) + } +} + +// signalSender wakes the sender's wait: a pickup, an update, the end of +// the task or an abort +func (t *workerTask) signalSender() { + C.frankenphp_task_chan_signal(C.intptr_t(t.fds[0]), C.intptr_t(t.fds[1]), 0) +} + +// signalReceiver wakes the receiver's stream_select(): the sender is gone +func (t *workerTask) signalReceiver() { + C.frankenphp_task_chan_signal(C.intptr_t(t.fds[0]), C.intptr_t(t.fds[1]), 1) +} + +// retire counts a side done with the task; the last one frees it +func (t *workerTask) retire() { + t.mu.Lock() + t.retired++ + last := t.retired == 2 + t.mu.Unlock() + + if last { + t.free() + } +} + +// free releases whatever the task still holds: called by the last side to +// close its stream, or by the sender when no thread picked the task up +func (t *workerTask) free() { + if t.payload != nil { + C.frankenphp_vars_free(t.payload) + } + for _, update := range t.updates { + C.frankenphp_vars_free(update) + } + taskChanPut(t.fds) + t.handle.Delete() +} + +//export go_frankenphp_send_task +func go_frankenphp_send_task(threadIndex C.uintptr_t, name *C.char, nameLen C.size_t, payload *C.HashTable) (C.uintptr_t, C.intptr_t, *C.char) { + thread := phpThreads[threadIndex] + workerName := C.GoStringN(name, C.int(nameLen)) + w := backgroundWorkerByName(thread.handler.frankenPHPContext(), workerName) + if w == nil { + C.frankenphp_vars_free(payload) + + return 0, -1, C.CString("frankenphp_send_task(): unknown background worker " + strconv.Quote(workerName)) + } + if handler, ok := thread.handler.(*backgroundWorkerThread); ok && handler.worker == w && w.countThreads() == 1 { + C.frankenphp_vars_free(payload) + + return 0, -1, C.CString("frankenphp_send_task(): background worker " + strconv.Quote(workerName) + " has a single thread and cannot send a task to itself") + } + // closed when this thread is drained for a restart or the shutdown: the + // target's threads are drained too, nobody would pick the task up + drainChan := thread.drainChan + + fds, ok := taskChanGet() + if !ok { + C.frankenphp_vars_free(payload) + + return 0, -1, C.CString("frankenphp_send_task(): failed to create the channel of the task") + } + + t := &workerTask{ + worker: w, + payload: payload, + pickedUp: make(chan struct{}), + cancelled: make(chan struct{}), + fds: fds, + } + t.cond = sync.NewCond(&t.mu) + t.handle = cgo.NewHandle(t) + + // queued like a request would be: a background worker has no other queue + metrics.QueuedWorkerRequest(w.qualifiedName) + q := &w.tasks + q.mu.Lock() + q.pending = append(q.pending, t) + handler, sock := w.claimParkedThread() + q.mu.Unlock() + if handler != nil { + signalThreads([]*backgroundWorkerThread{handler}, []int64{sock}) + } + + // the C side waits for the pickup on the sender's descriptor, in the + // kernel rather than in a Go select: waking a thread parked inside a Go + // callback costs the scheduler a hand-off, a signal on a descriptor does + // not. The thread taking the task sends it, the watcher does when the + // wait must end without a pickup. The shutdown channel is read here, on + // the PHP thread: the goroutine may only get to run after Shutdown() + go t.watch(drainChan, mainThread.done) + + return C.uintptr_t(t.handle), C.intptr_t(t.fds[0]), nil +} + +// watch escalates the wake-up when the thread signaled first does not come +// and ends the sender's wait when its thread is drained or FrankenPHP shuts +// down; it returns once the task is picked up or the sender gave up +func (t *workerTask) watch(drainChan, shutdown <-chan struct{}) { + escalate := time.NewTimer(taskSignalEscalation) + defer escalate.Stop() + + for { + select { + case <-t.pickedUp: + return + case <-t.cancelled: + return + case <-escalate.C: + q := &t.worker.tasks + q.mu.Lock() + var handlers []*backgroundWorkerThread + var socks []int64 + if slices.Contains(q.pending, t) { + handlers, socks = t.worker.claimAllThreads() + } + q.mu.Unlock() + signalThreads(handlers, socks) + case <-drainChan: + t.abort("frankenphp_send_task(): the calling thread is restarting or shutting down") + + return + case <-shutdown: + t.abort("frankenphp_send_task(): FrankenPHP is shutting down") + + return + } + } +} + +// abort ends the sender's wait for a pickup that must not happen anymore +func (t *workerTask) abort(reason string) { + q := &t.worker.tasks + q.mu.Lock() + if slices.Contains(q.pending, t) { + t.abortReason = reason + t.signalSender() + } + q.mu.Unlock() +} + +// go_frankenphp_task_side_gone tells a stream whether the other side closed +// its own: what feof() reports on the task streams +// +//export go_frankenphp_task_side_gone +func go_frankenphp_task_side_gone(handle C.uintptr_t, sender C.bool) C.bool { + t := cgo.Handle(handle).Value().(*workerTask) + + t.mu.Lock() + defer t.mu.Unlock() + if bool(sender) { + return C.bool(t.closed) + } + + return C.bool(t.senderGone) +} + +// go_frankenphp_task_await tells the sender, woken on its socket, where its +// task stands: 1 picked up, 2 aborted with the reason, 0 neither +// +//export go_frankenphp_task_await +func go_frankenphp_task_await(handle C.uintptr_t) (C.int, *C.char) { + t := cgo.Handle(handle).Value().(*workerTask) + + select { + case <-t.pickedUp: + return 1, nil + default: + } + + q := &t.worker.tasks + q.mu.Lock() + reason := t.abortReason + q.mu.Unlock() + if reason != "" { + return 2, C.CString(reason) + } + + return 0, nil +} + +// go_frankenphp_task_cancel takes a task nobody picked up out of the queue +// and releases the receiver's side of it, the sender's stream close releases +// the rest; false when a thread got the task first +// +//export go_frankenphp_task_cancel +func go_frankenphp_task_cancel(handle C.uintptr_t, timedOut C.bool) C.bool { + t := cgo.Handle(handle).Value().(*workerTask) + if !t.worker.tasks.remove(t) { + return false + } + close(t.cancelled) + + name := t.worker.qualifiedName + metrics.DequeuedWorkerRequest(name) + if bool(timedOut) { + metrics.WorkerTaskOutcome(name, TaskOutcomeTimeout) + } + + C.frankenphp_vars_free(t.payload) + t.payload = nil + t.mu.Lock() + // nothing for the sender's close to settle + t.closed = true + t.mu.Unlock() + t.retire() + + return true +} + +// go_frankenphp_background_worker_park is called by frankenphp_worker_tick() +// as the script is about to wait on its handle: the thread parks unless +// tasks are queued, in which case a wake-up is written on its own handle so +// the wait returns at once and the script dequeues them. Under tasks.mu, so +// a task queued after the check finds the thread parked and signals it: no +// wake-up is lost either way. The flag stays set when the wait returns for +// another reason than a claim: a claim meanwhile writes a wake-up the +// script's next wait returns on. +// +//export go_frankenphp_background_worker_park +func go_frankenphp_background_worker_park(threadIndex C.uintptr_t) { + handler, ok := phpThreads[threadIndex].handler.(*backgroundWorkerThread) + if !ok { + return + } + + q := &handler.worker.tasks + q.mu.Lock() + defer q.mu.Unlock() + + if len(q.pending) == 0 { + handler.parked = true + + return + } + if handler.stopSock >= 0 { + C.frankenphp_worker_signal_task(C.intptr_t(handler.stopSock)) + } +} + +//export go_frankenphp_receive_task +func go_frankenphp_receive_task(threadIndex C.uintptr_t) (C.uintptr_t, *C.HashTable, C.intptr_t) { + handler, ok := phpThreads[threadIndex].handler.(*backgroundWorkerThread) + if !ok { + // refused on the C side already + return 0, nil, -1 + } + + q := &handler.worker.tasks + q.mu.Lock() + if len(q.pending) == 0 { + q.mu.Unlock() + + return 0, nil, -1 + } + t := q.pending[0] + q.pending = slices.Delete(q.pending, 0, 1) + // the payload moves to request memory on the C side + payload := t.payload + t.payload = nil + q.mu.Unlock() + metrics.DequeuedWorkerRequest(handler.worker.qualifiedName) + close(t.pickedUp) + // wakes the sender's wait for the pickup, see go_frankenphp_send_task; + // after the channel, so the sender finds it closed once woken + t.signalSender() + + t.receiver = handler + t.pickedUpAt = time.Now() + metrics.StartWorkerTask(handler.worker.qualifiedName) + // busy on the threads endpoint while it holds a task + if handler.openTasks++; handler.openTasks == 1 { + handler.state.MarkAsWaiting(false) + } + + return C.uintptr_t(t.handle), payload, C.intptr_t(t.fds[1]) +} + +//export go_frankenphp_update_task +func go_frankenphp_update_task(handle C.uintptr_t, update *C.HashTable) *C.char { + t := cgo.Handle(handle).Value().(*workerTask) + + t.mu.Lock() + for len(t.updates) >= taskUpdatesMax && !t.senderGone { + t.cond.Wait() + } + if t.senderGone { + t.mu.Unlock() + C.frankenphp_vars_free(update) + + return C.CString("frankenphp_update_task(): the sender closed the task") + } + t.updates = append(t.updates, update) + t.mu.Unlock() + + // one signal per update, after the push: the sender consumes one per + // update it reads + t.signalSender() + + return nil +} + +//export go_frankenphp_read_task +func go_frankenphp_read_task(handle C.uintptr_t) (*C.HashTable, C.int) { + t := cgo.Handle(handle).Value().(*workerTask) + + t.mu.Lock() + defer t.mu.Unlock() + + if len(t.updates) > 0 { + update := t.updates[0] + t.updates = slices.Delete(t.updates, 0, 1) + t.cond.Signal() + + return update, C.int(C.FRANKENPHP_TASK_READ_UPDATE) + } + switch { + case t.aborted: + return nil, C.int(C.FRANKENPHP_TASK_READ_ABORTED) + case t.closed: + return nil, C.int(C.FRANKENPHP_TASK_READ_COMPLETED) + } + + return nil, C.int(C.FRANKENPHP_TASK_READ_PENDING) +} + +//export go_frankenphp_task_receiver_close +func go_frankenphp_task_receiver_close(handle C.uintptr_t, aborted C.bool) { + t := cgo.Handle(handle).Value().(*workerTask) + + t.mu.Lock() + t.closed = true + t.aborted = bool(aborted) + // the first side to close settles the outcome + settled := !t.senderGone + t.cond.Broadcast() + t.mu.Unlock() + // the sender finds the end of the task behind the updates still queued; + // nobody waits on its descriptor once it closed + if settled { + t.signalSender() + } + + name := t.worker.qualifiedName + metrics.StopWorkerTask(name, time.Since(t.pickedUpAt)) + if settled { + outcome := TaskOutcomeCompleted + if aborted { + outcome = TaskOutcomeAborted + } + metrics.WorkerTaskOutcome(name, outcome) + } + handler := t.receiver + handler.openTasks-- + if handler.openTasks == 0 && !handler.isBootingScript { + handler.state.MarkAsWaiting(true) + } + + t.retire() +} + +//export go_frankenphp_task_sender_close +func go_frankenphp_task_sender_close(handle C.uintptr_t) { + t := cgo.Handle(handle).Value().(*workerTask) + + t.mu.Lock() + t.senderGone = true + // the first side to close settles the outcome + settled := !t.closed + updates := t.updates + t.updates = nil + t.cond.Broadcast() + t.mu.Unlock() + + if settled { + // the receiver's stream_select() and feof() see it; once the + // receiver closed, nobody waits on its descriptor + t.signalReceiver() + metrics.WorkerTaskOutcome(t.worker.qualifiedName, TaskOutcomeAbandoned) + } + for _, update := range updates { + C.frankenphp_vars_free(update) + } + t.retire() +} diff --git a/workertask_test.go b/workertask_test.go new file mode 100644 index 0000000000..041ac921d9 --- /dev/null +++ b/workertask_test.go @@ -0,0 +1,247 @@ +package frankenphp_test + +import ( + "io" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/dunglas/frankenphp" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTaskRoundTrip(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("echo", "task-worker.php", nil, server), frankenphp.WithNumThreads(2)) + + body := serverGet(t, server, "http://example.com/task.php?input=hello") + assert.Contains(t, body, `"result":"processed:hello"`) + assert.Contains(t, body, `"worker":"echo"`) + assert.True(t, strings.HasSuffix(body, "\ndone"), body) + + // the worker loops: a second task on the same thread + assert.Contains(t, serverGet(t, server, "http://example.com/task.php?input=again"), `"result":"processed:again"`) + + // progress updates come in order, before the result + lines := strings.Split(serverGet(t, server, "http://example.com/task.php?input=steps&steps=2"), "\n") + require.Len(t, lines, 4) + assert.Equal(t, `{"step":1,"of":2}`, lines[0]) + assert.Equal(t, `{"step":2,"of":2}`, lines[1]) + assert.Contains(t, lines[2], `"result":"processed:steps"`) + assert.Equal(t, "done", lines[3]) +} + +func TestTaskScopedToServer(t *testing.T) { + server1, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("one")) + server2, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("two")) + initServers(t, + frankenphp.WithServer(server1), + frankenphp.WithServer(server2), + bgWorker("jobs", "task-worker.php", map[string]string{"BG_TAG": "one"}, server1), + bgWorker("jobs", "task-worker.php", map[string]string{"BG_TAG": "two"}, server2), + frankenphp.WithNumThreads(3), + ) + + assert.Contains(t, serverGet(t, server1, "http://example.com/task.php?name=jobs"), `"tag":"one"`) + assert.Contains(t, serverGet(t, server2, "http://example.com/task.php?name=jobs"), `"tag":"two"`) +} + +// a background worker may send tasks too, here while booting +func TestTaskFromBackgroundWorker(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "relay.json") + initServers(t, + bgWorker("relay", "task-relay.php", map[string]string{"BG_TARGET": "echo", "BG_SENTINEL": sentinel}, nil), + bgWorker("echo", "task-worker.php", nil, nil), + frankenphp.WithNumThreads(3), + ) + + assert.Contains(t, requireFileContentEventually(t, sentinel), `"result":"processed:relayed"`) +} + +// a sender waits for a thread to pick its task up, up to the timeout +func TestTaskPickupTimeout(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("echo", "task-worker.php", nil, server), frankenphp.WithNumThreads(2)) + + body := serverGet(t, server, "http://example.com/task-busy.php") + assert.Contains(t, body, `picked up the task in time`) + assert.Contains(t, body, `"result":"processed:slow"`) +} + +// a worker exiting with a task open fails the sender's read, then restarts +func TestTaskCrashMidTask(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("echo", "task-worker.php", nil, server), frankenphp.WithNumThreads(2)) + + assert.Contains(t, serverGet(t, server, "http://example.com/task.php?crash=1"), "exited without completing the task") + assert.Contains(t, serverGet(t, server, "http://example.com/task.php?input=after"), `"result":"processed:after"`) +} + +// closing the stream abandons the task: the worker sees it on its own stream +func TestTaskAbandoned(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "abandoned.txt") + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("echo", "task-worker.php", map[string]string{"BG_SENTINEL": sentinel}, server), frankenphp.WithNumThreads(2)) + + assert.Equal(t, "closed", serverGet(t, server, "http://example.com/task.php?sleep_ms=200&close_early=1")) + assert.Contains(t, requireFileContentEventually(t, sentinel), "the sender closed the task before the update") +} + +// a task queued while the only thread is busy reaches it when it reads its +// handle again, even with a loop taking one task per wake-up +func TestTaskQueuedWhileBusy(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("echo", "task-worker.php", map[string]string{"BG_LOOP": "if"}, server), frankenphp.WithNumThreads(3)) + + bodies := make(chan string, 2) + for _, input := range []string{"first", "second"} { + go func() { + w := httptest.NewRecorder() + _ = server.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "http://example.com/task.php?sleep_ms=100&input="+input, nil)) + b, _ := io.ReadAll(w.Result().Body) + bodies <- string(b) + }() + } + results := <-bodies + <-bodies + assert.Contains(t, results, `"result":"processed:first"`) + assert.Contains(t, results, `"result":"processed:second"`) +} + +// the threads of a pool share the queue, and stream_select() works on the +// sender's streams +func TestTaskPool(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + frankenphp.WithWorkers("pool", "testdata/bgworker/task-worker.php", 2, frankenphp.WithWorkerBackground(), frankenphp.WithWorkerServerScope(server)), + frankenphp.WithNumThreads(3), + ) + + assert.Equal(t, "[\"processed:a\",\"processed:b\"]\ntwo threads", serverGet(t, server, "http://example.com/task-pool.php")) +} + +func TestTaskErrors(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("echo", "task-worker.php", nil, server), frankenphp.WithNumThreads(2)) + + body := serverGet(t, server, "http://example.com/task-errors.php") + assert.Contains(t, body, `unknown: RuntimeException: frankenphp_send_task(): unknown background worker "nope"`) + assert.Contains(t, body, "payload: ValueError: frankenphp_send_task(): payload values must be null, scalars, arrays or enums") + assert.Contains(t, body, "timeout: ValueError: frankenphp_send_task(): Argument #3 ($timeout) must be greater than or equal to 0") + assert.Contains(t, body, "receive: RuntimeException: frankenphp_receive_task() can only be called from a background worker") + assert.Contains(t, body, "update: TypeError: frankenphp_update_task(): Argument #1 ($stream) must be a stream returned by frankenphp_receive_task()") + assert.Contains(t, body, "read: TypeError: frankenphp_read_task(): Argument #1 ($stream) must be a stream returned by frankenphp_send_task()") +} + +// the metrics of a background worker follow its tasks: busy while a thread +// holds one, queued while nobody picked it up, counted by outcome +func TestTaskMetrics(t *testing.T) { + registry := prometheus.NewRegistry() + sentinel := filepath.Join(t.TempDir(), "abandoned.txt") + server, err := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("api")) + require.NoError(t, err) + initServers(t, + frankenphp.WithServer(server), + bgWorker("echo", "task-worker.php", map[string]string{"BG_SENTINEL": sentinel}, server), + frankenphp.WithNumThreads(2), + frankenphp.WithMetrics(frankenphp.NewPrometheusMetrics(registry)), + ) + + assert.Contains(t, serverGet(t, server, "http://example.com/task.php?input=done"), `"result":"processed:done"`) + assert.Contains(t, serverGet(t, server, "http://example.com/task.php?crash=1"), "exited without completing the task") + assert.Equal(t, "closed", serverGet(t, server, "http://example.com/task.php?sleep_ms=200&close_early=1")) + requireFileContentEventually(t, sentinel) + assert.Contains(t, serverGet(t, server, "http://example.com/task-busy.php"), "picked up the task in time") + + expected := ` + # HELP frankenphp_worker_task_count Number of tasks sent to this background worker, by outcome: completed, aborted (the script ended with the task open), abandoned (the sender closed its stream first) or timeout (no thread picked the task up in time) + # TYPE frankenphp_worker_task_count counter + frankenphp_worker_task_count{outcome="abandoned",worker="api:echo"} 1 + frankenphp_worker_task_count{outcome="aborted",worker="api:echo"} 1 + frankenphp_worker_task_count{outcome="completed",worker="api:echo"} 2 + frankenphp_worker_task_count{outcome="timeout",worker="api:echo"} 1 + # HELP frankenphp_busy_workers Number of busy PHP workers for this worker: processing a request, or a task for a background worker + # TYPE frankenphp_busy_workers gauge + frankenphp_busy_workers{worker="api:echo"} 0 + # HELP frankenphp_worker_queue_depth Number of queued requests for this worker, or of tasks waiting for a thread of a background worker + # TYPE frankenphp_worker_queue_depth gauge + frankenphp_worker_queue_depth{worker="api:echo"} 0 + ` + // the abandoned task is closed by the worker after the response + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.NoError(c, testutil.GatherAndCompare(registry, strings.NewReader(expected), "frankenphp_worker_task_count", "frankenphp_busy_workers", "frankenphp_worker_queue_depth")) + }, 5*time.Second, 25*time.Millisecond) +} + +// a sender waiting for a busy worker to pick its task up is released by +// Shutdown() instead of holding it +func TestTaskSenderUnblockedOnShutdown(t *testing.T) { + mark := filepath.Join(t.TempDir(), "picked") + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + t.Cleanup(frankenphp.Shutdown) + require.NoError(t, frankenphp.Init(frankenphp.WithServer(server), bgWorker("echo", "task-worker.php", nil, server), frankenphp.WithNumThreads(2))) + + body := sendWhileWorkerBusy(t, server, mark) + done := make(chan struct{}) + go func() { + frankenphp.Shutdown() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Shutdown did not return within 10s") + } + assert.Contains(t, <-body, "FrankenPHP is shutting down") +} + +// a restart drains the sender's thread too: the wait for a pickup ends +// instead of stalling the restart until the timeout +func TestTaskSenderUnblockedOnRestart(t *testing.T) { + mark := filepath.Join(t.TempDir(), "picked") + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("echo", "task-worker.php", nil, server), frankenphp.WithNumThreads(2)) + + body := sendWhileWorkerBusy(t, server, mark) + start := time.Now() + frankenphp.RestartWorkers() + assert.WithinDuration(t, start, time.Now(), 10*time.Second, "the restart must not wait for the sender's timeout") + assert.Contains(t, <-body, "the calling thread is restarting or shutting down") + + // the restarted worker serves tasks again + assert.Contains(t, serverGet(t, server, "http://example.com/task.php?input=after"), `"result":"processed:after"`) +} + +// sendWhileWorkerBusy runs task-shutdown.php in the background: its first +// task keeps the only thread of the worker busy, its second one has no +// timeout; returns the channel carrying the response body once the first +// task was picked up +func sendWhileWorkerBusy(t *testing.T, server *frankenphp.Server, mark string) <-chan string { + t.Helper() + body := make(chan string, 1) + go func() { + w := httptest.NewRecorder() + _ = server.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "http://example.com/task-shutdown.php?mark="+url.QueryEscape(mark), nil)) + b, _ := io.ReadAll(w.Result().Body) + body <- string(b) + }() + requireFileEventually(t, mark, "the worker did not pick the first task up") + + return body +} diff --git a/workervars.go b/workervars.go new file mode 100644 index 0000000000..1e7a09cb7c --- /dev/null +++ b/workervars.go @@ -0,0 +1,156 @@ +package frankenphp + +// #include "frankenphp.h" +import "C" +import ( + "errors" + "strconv" + "sync" +) + +// varsSlot holds the snapshot a background worker published through +// frankenphp_set_vars(): a persistent HashTable, copied into request memory +// by each frankenphp_get_vars() reader. It belongs to the worker, not to a +// thread, so it survives script restarts and serves stale data meanwhile. +type varsSlot struct { + mu sync.RWMutex + table *C.HashTable +} + +var ( + // booting background workers blocked in frankenphp_get_vars() on other + // workers, keyed by waiter: a cycle would deadlock Init(), refuse it + varsWaitMu sync.Mutex + varsWaitOn = map[*worker]map[*worker]int{} +) + +// backgroundWorkerByName resolves a background worker the way requests +// resolve workers: within the caller's server first, then among global ones +func backgroundWorkerByName(fc *frankenPHPContext, name string) *worker { + var w *worker + if fc != nil && fc.server != nil { + w = fc.server.workersByName[name] + } + if w == nil { + w = fallbackServer.workersByName[name] + } + if w == nil || !w.isBackgroundWorker { + return nil + } + + return w +} + +// waitVarsReady blocks until target reached its ready point once. Requests +// cannot get here before that (activateServers() runs after initWorkers()), +// so a blocked caller is a background worker still booting: waits between +// workers form a graph, and a cycle is refused instead of deadlocking Init() +func waitVarsReady(target, caller *worker) error { + select { + case <-target.readyOnce: + return nil + default: + } + + if caller != nil { + varsWaitMu.Lock() + if caller == target || varsWaitReaches(target, caller) { + varsWaitMu.Unlock() + + return errors.New("frankenphp_get_vars(): circular dependency between background workers " + strconv.Quote(caller.name) + " and " + strconv.Quote(target.name)) + } + if varsWaitOn[caller] == nil { + varsWaitOn[caller] = map[*worker]int{} + } + varsWaitOn[caller][target]++ + varsWaitMu.Unlock() + + defer func() { + varsWaitMu.Lock() + if varsWaitOn[caller][target]--; varsWaitOn[caller][target] == 0 { + delete(varsWaitOn[caller], target) + } + if len(varsWaitOn[caller]) == 0 { + delete(varsWaitOn, caller) + } + varsWaitMu.Unlock() + }() + } + + select { + case <-target.readyOnce: + return nil + case <-mainThread.done: + return errors.New("frankenphp_get_vars(): FrankenPHP is shutting down") + } +} + +// varsWaitReaches reports whether from waits, transitively, on to; called +// with varsWaitMu held +func varsWaitReaches(from, to *worker) bool { + for next := range varsWaitOn[from] { + if next == to || varsWaitReaches(next, to) { + return true + } + } + + return false +} + +// freeWorkerVars releases the snapshots once no PHP thread can read them +// and before the engine goes away +func freeWorkerVars() { + for _, w := range workers { + w.vars.mu.Lock() + if w.vars.table != nil { + C.frankenphp_vars_free(w.vars.table) + w.vars.table = nil + } + w.vars.mu.Unlock() + } +} + +//export go_frankenphp_set_vars +func go_frankenphp_set_vars(threadIndex C.uintptr_t, table *C.HashTable) *C.HashTable { + handler, ok := phpThreads[threadIndex].handler.(*backgroundWorkerThread) + if !ok { + // already refused on the C side; hand the table back so it is freed + return table + } + + slot := &handler.worker.vars + slot.mu.Lock() + old := slot.table + slot.table = table + slot.mu.Unlock() + + return old +} + +//export go_frankenphp_get_vars +func go_frankenphp_get_vars(threadIndex C.uintptr_t, name *C.char, nameLen C.size_t, returnValue *C.zval) *C.char { + thread := phpThreads[threadIndex] + workerName := C.GoStringN(name, C.int(nameLen)) + target := backgroundWorkerByName(thread.handler.frankenPHPContext(), workerName) + if target == nil { + return C.CString("frankenphp_get_vars(): unknown background worker " + strconv.Quote(workerName)) + } + + var caller *worker + if handler, ok := thread.handler.(*backgroundWorkerThread); ok && handler.isBootingScript { + caller = handler.worker + } + if err := waitVarsReady(target, caller); err != nil { + return C.CString(err.Error()) + } + + slot := &target.vars + slot.mu.RLock() + defer slot.mu.RUnlock() + if slot.table == nil { + return C.CString("frankenphp_get_vars(): background worker " + strconv.Quote(target.name) + " has not published any vars yet") + } + C.frankenphp_vars_to_request(returnValue, slot.table) + + return nil +} diff --git a/workervars_test.go b/workervars_test.go new file mode 100644 index 0000000000..565166eddc --- /dev/null +++ b/workervars_test.go @@ -0,0 +1,95 @@ +package frankenphp_test + +import ( + "path/filepath" + "testing" + + "github.com/dunglas/frankenphp" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// bgWorker declares a background worker from testdata/bgworker, scoped to +// server when one is given +func bgWorker(name, file string, env map[string]string, server *frankenphp.Server) frankenphp.Option { + opts := []frankenphp.WorkerOption{frankenphp.WithWorkerBackground(), frankenphp.WithWorkerEnv(env)} + if server != nil { + opts = append(opts, frankenphp.WithWorkerServerScope(server)) + } + + return frankenphp.WithWorkers(name, "testdata/bgworker/"+file, 1, opts...) +} + +func TestVarsRoundTrip(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("publisher", "publisher.php", nil, server), frankenphp.WithNumThreads(2)) + + body := serverGet(t, server, "http://example.com/vars.php?name=publisher") + assert.JSONEq(t, `{"answer":42,"value":"default","nested":{"a":1,"list":[true,null,1.5,"x"]},"worker":"publisher"}`, body) + + // a second read is a fresh copy of the same snapshot + assert.JSONEq(t, body, serverGet(t, server, "http://example.com/vars.php?name=publisher")) + assert.Contains(t, serverGet(t, server, "http://example.com/vars.php?name=nope"), "unknown background worker") + assert.Contains(t, serverGet(t, server, "http://example.com/set-vars-outside.php"), "can only be called from a background worker") +} + +func TestVarsScopedToServer(t *testing.T) { + server1, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("one")) + server2, _ := frankenphp.NewServer(testDataDir, frankenphp.WithServerName("two")) + initServers(t, + frankenphp.WithServer(server1), + frankenphp.WithServer(server2), + bgWorker("cfg", "publisher.php", map[string]string{"BG_PUBLISH_VALUE": "one"}, server1), + bgWorker("cfg", "publisher.php", map[string]string{"BG_PUBLISH_VALUE": "two"}, server2), + frankenphp.WithNumThreads(3), + ) + + assert.Contains(t, serverGet(t, server1, "http://example.com/vars.php?name=cfg"), `"value":"one"`) + assert.Contains(t, serverGet(t, server2, "http://example.com/vars.php?name=cfg"), `"value":"two"`) +} + +// a worker booting before its dependency has published blocks in +// frankenphp_get_vars() until the dependency is ready +func TestVarsBlockUntilReady(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "consumer.json") + initServers(t, + bgWorker("consumer", "consumer.php", map[string]string{"BG_CONSUME": "publisher", "BG_SENTINEL": sentinel}, nil), + bgWorker("publisher", "publisher.php", map[string]string{"BG_PUBLISH_DELAY_MS": "500"}, nil), + frankenphp.WithNumThreads(3), + ) + + assert.Contains(t, requireFileContentEventually(t, sentinel), `"answer":42`) +} + +func TestVarsCycleIsRefused(t *testing.T) { + tmp := t.TempDir() + s1, s2 := filepath.Join(tmp, "c1.txt"), filepath.Join(tmp, "c2.txt") + initServers(t, + bgWorker("c1", "consumer.php", map[string]string{"BG_CONSUME": "c2", "BG_SENTINEL": s1}, nil), + bgWorker("c2", "consumer.php", map[string]string{"BG_CONSUME": "c1", "BG_SENTINEL": s2}, nil), + frankenphp.WithNumThreads(3), + ) + + // one side sees the cycle, the other then reads a ready worker that never published + results := requireFileContentEventually(t, s1) + "\n" + requireFileContentEventually(t, s2) + assert.Contains(t, results, "circular dependency") + assert.Contains(t, results, "has not published any vars yet") +} + +func TestVarsNotPublished(t *testing.T) { + server, err := frankenphp.NewServer(testDataDir) + require.NoError(t, err) + initServers(t, frankenphp.WithServer(server), bgWorker("silent", "basic.php", nil, server), frankenphp.WithNumThreads(2)) + + assert.Contains(t, serverGet(t, server, "http://example.com/vars.php?name=silent"), "has not published any vars yet") +} + +func TestVarsRejectInvalidValues(t *testing.T) { + sentinel := filepath.Join(t.TempDir(), "bad.txt") + initServers(t, bgWorker("bad", "bad-vars.php", map[string]string{"BG_SENTINEL": sentinel}, nil), frankenphp.WithNumThreads(2)) + + result := requireFileContentEventually(t, sentinel) + assert.Contains(t, result, "ValueError") + assert.Contains(t, result, "must be null, scalars, arrays or enums") +} diff --git a/zval.h b/zval.h index f75cccdc60..cb165670d4 100644 --- a/zval.h +++ b/zval.h @@ -12,7 +12,8 @@ * one request). * * Fast paths: - * - Interned strings: shared memory, no copy. + * - Permanent interned strings (opcache, startup): shared, no copy. + * Strings interned during a request die with it, they are copied. * - Opcache-immutable arrays: shared pointer, no copy, no free. * * Included by frankenphp.c; not a standalone compilation unit. */ @@ -81,6 +82,12 @@ static bool persistent_zval_validate(zval *z) { return persistent_zval_validate_depth(z, 0); } +/* Only permanent interned strings outlive the request that interned them: + * those are the ones a persistent tree may share by pointer. */ +static bool persistent_zval_str_is_shared(zend_string *s) { + return ZSTR_IS_INTERNED(s) && (GC_FLAGS(s) & IS_STR_PERMANENT) != 0; +} + /* Deep-copy a zval from request memory into persistent (pemalloc) memory. * Callers must have already passed persistent_zval_validate on src. * @@ -103,8 +110,8 @@ static void persistent_zval_persist(zval *dst, zval *src) { break; case IS_STRING: { zend_string *s = Z_STR_P(src); - if (ZSTR_IS_INTERNED(s)) { - ZVAL_STR(dst, s); /* interned strings live process-wide */ + if (persistent_zval_str_is_shared(s)) { + ZVAL_STR(dst, s); } else { ZVAL_NEW_STR(dst, zend_string_init(ZSTR_VAL(s), ZSTR_LEN(s), 1)); } @@ -115,13 +122,13 @@ static void persistent_zval_persist(zval *dst, zval *src) { zend_class_entry *ce = Z_OBJCE_P(src); persistent_zval_enum_t *e = pemalloc(sizeof(*e), 1); e->class_name = - ZSTR_IS_INTERNED(ce->name) + persistent_zval_str_is_shared(ce->name) ? ce->name : zend_string_init(ZSTR_VAL(ce->name), ZSTR_LEN(ce->name), 1); zval *case_name_zval = zend_enum_fetch_case_name(Z_OBJ_P(src)); zend_string *case_str = Z_STR_P(case_name_zval); e->case_name = - ZSTR_IS_INTERNED(case_str) + persistent_zval_str_is_shared(case_str) ? case_str : zend_string_init(ZSTR_VAL(case_str), ZSTR_LEN(case_str), 1); ZVAL_PTR(dst, e); @@ -131,8 +138,10 @@ static void persistent_zval_persist(zval *dst, zval *src) { HashTable *src_ht = Z_ARRVAL_P(src); if ((GC_FLAGS(src_ht) & IS_ARRAY_IMMUTABLE) != 0) { /* Opcache-immutable arrays live for the process lifetime and are - * safe to share across threads by pointer. Zero-copy, zero-free. */ + * safe to share across threads by pointer. Zero-copy, zero-free. + * Not refcounted: the zval must not count on the array. */ ZVAL_ARR(dst, src_ht); + Z_TYPE_FLAGS_P(dst) = 0; break; } HashTable *dst_ht = pemalloc(sizeof(HashTable), 1); @@ -146,7 +155,7 @@ static void persistent_zval_persist(zval *dst, zval *src) { zval pval; persistent_zval_persist(&pval, val); if (key) { - if (ZSTR_IS_INTERNED(key)) { + if (persistent_zval_str_is_shared(key)) { zend_hash_add_new(dst_ht, key, &pval); } else { zend_string *pkey = zend_string_init(ZSTR_VAL(key), ZSTR_LEN(key), 1); @@ -258,8 +267,12 @@ static void persistent_zval_to_request(zval *dst, zval *src) { case IS_ARRAY: { HashTable *src_ht = Z_ARRVAL_P(src); if ((GC_FLAGS(src_ht) & IS_ARRAY_IMMUTABLE) != 0) { - /* Zero-copy: immutable arrays are safe to expose directly. */ + /* Zero-copy: immutable arrays are safe to expose directly, as long + * as the zval does not count on them: opcache keeps their refcount + * at 2 as a safety net, so a refcounted zval exposing the same array + * twice would destroy shared memory on the second release. */ ZVAL_ARR(dst, src_ht); + Z_TYPE_FLAGS_P(dst) = 0; break; } array_init_size(dst, zend_hash_num_elements(src_ht)); diff --git a/zval_test.go b/zval_test.go index 80aad7f16f..e25903b0b7 100644 --- a/zval_test.go +++ b/zval_test.go @@ -47,4 +47,5 @@ func TestPersistentZvalRoundtrip(t *testing.T) { require.Contains(t, out, "OK stdClass rejected") require.Contains(t, out, "OK resource rejected") require.Contains(t, out, "OK nested stdClass rejected") + require.Contains(t, out, "OK immutable literal exposed repeatedly") }