From ab594a0b9a70fbd4cff259e9d6485bbb0768bf94 Mon Sep 17 00:00:00 2001 From: Nicolas Grekas Date: Sun, 6 Sep 2026 14:59:08 +0200 Subject: [PATCH 1/3] fix: log opcache restarts opcache schedules a restart of its shared memory on exhaustion or hash overflow, then carries it out at the next request init on any thread. Under ZTS it does that while other threads are still running, because the deferral gate (accel_is_inactive()) probes for a conflicting lock with fcntl F_GETLK, and POSIX fcntl locks belong to the process, so the probe never sees the threads of the process holding them. Workers are the worst case: they hold shared memory references for their whole life rather than for a single request. The result is a crash or a slowdown with nothing in the logs pointing at opcache. opcache does report it, but only at opcache.log_verbosity_level=4 and in its own log. zend_accel_schedule_restart_hook is the only in-process signal for this, so it is used to emit one warning naming the restart reason and the two settings that make restarts less likely. Nothing else is done with it: the threads are not rebooted, which is what #2564 removed. --- frankenphp.c | 13 +++++++++++++ frankenphp.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/frankenphp.c b/frankenphp.c index b15507f69d..0be2f79dad 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -1017,6 +1017,14 @@ PHP_FUNCTION(frankenphp_log) { } } +/* Guarded like its only assignment in php_main(), so builds without the hook + * do not trip -Werror=unused-function. */ +#if defined(ZTS) && PHP_VERSION_ID >= 80400 +static void frankenphp_opcache_restart_hook(int reason) { + go_log_opcache_restart(reason); +} +#endif + /* {{{ thread-safe opcache reset */ PHP_FUNCTION(frankenphp_opcache_reset) { go_schedule_opcache_reset(frankenphp_thread_index()); @@ -1705,6 +1713,11 @@ static void *php_main(void *arg) { frankenphp_sapi_module.startup(&frankenphp_sapi_module); +#if defined(ZTS) && PHP_VERSION_ID >= 80400 + /* Report the opcache restarts that opcache schedules on its own */ + zend_accel_schedule_restart_hook = frankenphp_opcache_restart_hook; +#endif + /* check if a default filter is set in php.ini and only filter if * it is, this is deprecated and will be removed in PHP 9 */ char *default_filter; diff --git a/frankenphp.go b/frankenphp.go index 79b135b808..cfc88a2050 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -780,6 +780,35 @@ func go_schedule_opcache_reset(threadIndex C.uintptr_t) { } } +// Restart reasons opcache reports to the hook, in the order of +// zend_accel_restart_reason (ext/opcache/ZendAccelerator.h). +var opcacheRestartReasons = [...]string{"out of memory", "hash overflow", "user"} + +// go_log_opcache_restart reports the restarts opcache schedules on its own. +// Under ZTS they rewind shared memory that running threads still point into, +// which surfaces as an unexplained crash or slowdown, so make the event +// visible. The line is written inline even though opcache can be holding its +// shared memory lock: that costs far less than the restart it precedes, and a +// line deferred to a goroutine would be lost when the restart takes the +// process down. +// +//export go_log_opcache_restart +func go_log_opcache_restart(reason C.int) { + if !globalLogger.Enabled(globalCtx, slog.LevelWarn) { + return + } + + reasonText := "unknown" + if i := int(reason); i >= 0 && i < len(opcacheRestartReasons) { + reasonText = opcacheRestartReasons[i] + } + + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, + "opcache restart scheduled, running PHP threads may hold stale references to its shared memory: raise opcache.memory_consumption and opcache.max_accelerated_files to make restarts less likely", + slog.String("reason", reasonText), + ) +} + func convertArgs(args []string) (C.int, []*C.char) { argc := C.int(len(args)) argv := make([]*C.char, argc) From 4486a7c97894dde54e5c6dde3e1a42becdbd83a7 Mon Sep 17 00:00:00 2001 From: Nicolas Grekas Date: Mon, 7 Sep 2026 11:01:29 +0200 Subject: [PATCH 2/3] feat: count opcache restarts in frankenphp_opcache_restarts The log line alone cannot be alerted on. Expose the same event as a counter labelled by reason, pre-populated at zero for the known reasons so a rate or an alert works from the first restart on. --- docs/metrics.md | 1 + frankenphp.c | 2 +- frankenphp.go | 22 ++++++++-------------- metrics.go | 28 ++++++++++++++++++++++++++++ metrics_test.go | 27 +++++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 15 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 932707265a..7a6a0981cf 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -23,6 +23,7 @@ When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenP - `frankenphp_worker_crashes{worker="[worker_name]"}`: The number of times a worker has unexpectedly terminated. - `frankenphp_worker_restarts{worker="[worker_name]"}`: The number of times a worker has been deliberately restarted. - `frankenphp_worker_queue_depth{worker="[worker_name]"}`: The number of queued requests. +- `frankenphp_opcache_restarts{reason="[reason]"}`: The number of times opcache restarted its shared memory on its own, by reason (`out of memory`, `hash overflow`, `user`). Each restart is also logged. Under ZTS, running PHP threads may hold stale references to the rewound memory, so raise `opcache.memory_consumption` and `opcache.max_accelerated_files` when this counter grows. For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used. diff --git a/frankenphp.c b/frankenphp.c index 0be2f79dad..f5d034694f 100644 --- a/frankenphp.c +++ b/frankenphp.c @@ -1021,7 +1021,7 @@ PHP_FUNCTION(frankenphp_log) { * do not trip -Werror=unused-function. */ #if defined(ZTS) && PHP_VERSION_ID >= 80400 static void frankenphp_opcache_restart_hook(int reason) { - go_log_opcache_restart(reason); + go_opcache_restart_scheduled(reason); } #endif diff --git a/frankenphp.go b/frankenphp.go index cfc88a2050..fb1dfaa6e6 100644 --- a/frankenphp.go +++ b/frankenphp.go @@ -784,25 +784,19 @@ func go_schedule_opcache_reset(threadIndex C.uintptr_t) { // zend_accel_restart_reason (ext/opcache/ZendAccelerator.h). var opcacheRestartReasons = [...]string{"out of memory", "hash overflow", "user"} -// go_log_opcache_restart reports the restarts opcache schedules on its own. -// Under ZTS they rewind shared memory that running threads still point into, -// which surfaces as an unexplained crash or slowdown, so make the event -// visible. The line is written inline even though opcache can be holding its -// shared memory lock: that costs far less than the restart it precedes, and a -// line deferred to a goroutine would be lost when the restart takes the -// process down. -// -//export go_log_opcache_restart -func go_log_opcache_restart(reason C.int) { - if !globalLogger.Enabled(globalCtx, slog.LevelWarn) { - return - } - +//export go_opcache_restart_scheduled +func go_opcache_restart_scheduled(reason C.int) { reasonText := "unknown" if i := int(reason); i >= 0 && i < len(opcacheRestartReasons) { reasonText = opcacheRestartReasons[i] } + metrics.OpcacheRestart(reasonText) + + if !globalLogger.Enabled(globalCtx, slog.LevelWarn) { + return + } + globalLogger.LogAttrs(globalCtx, slog.LevelWarn, "opcache restart scheduled, running PHP threads may hold stale references to its shared memory: raise opcache.memory_consumption and opcache.max_accelerated_files to make restarts less likely", slog.String("reason", reasonText), diff --git a/metrics.go b/metrics.go index baab7bbc90..5c9127649a 100644 --- a/metrics.go +++ b/metrics.go @@ -40,6 +40,8 @@ type Metrics interface { DequeuedWorkerRequest(name string) QueuedRequest() DequeuedRequest() + // OpcacheRestart collects the restarts of opcache's shared memory, by reason + OpcacheRestart(reason string) } type nullMetrics struct{} @@ -81,6 +83,8 @@ func (n nullMetrics) DequeuedWorkerRequest(string) {} func (n nullMetrics) QueuedRequest() {} func (n nullMetrics) DequeuedRequest() {} +func (n nullMetrics) OpcacheRestart(string) {} + type PrometheusMetrics struct { registry prometheus.Registerer totalThreads prometheus.Gauge @@ -94,6 +98,7 @@ type PrometheusMetrics struct { workerRequestCount *prometheus.CounterVec workerQueueDepth *prometheus.GaugeVec queueDepth prometheus.Gauge + opcacheRestarts *prometheus.CounterVec mu sync.RWMutex } @@ -332,6 +337,13 @@ func (m *PrometheusMetrics) DequeuedRequest() { m.queueDepth.Dec() } +func (m *PrometheusMetrics) OpcacheRestart(reason string) { + m.mu.RLock() + defer m.mu.RUnlock() + + m.opcacheRestarts.WithLabelValues(reason).Inc() +} + func (m *PrometheusMetrics) Shutdown() { m.mu.Lock() defer m.mu.Unlock() @@ -339,6 +351,7 @@ func (m *PrometheusMetrics) Shutdown() { m.registry.Unregister(m.totalThreads) m.registry.Unregister(m.busyThreads) m.registry.Unregister(m.queueDepth) + m.registry.Unregister(m.opcacheRestarts) if m.totalWorkers != nil { m.registry.Unregister(m.totalWorkers) @@ -392,6 +405,10 @@ func NewPrometheusMetrics(registry prometheus.Registerer) *PrometheusMetrics { Name: "frankenphp_queue_depth", Help: "Number of regular queued requests", }), + opcacheRestarts: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "frankenphp_opcache_restarts", + Help: "Number of restarts of opcache's shared memory, by reason", + }, []string{"reason"}), totalWorkers: nil, busyWorkers: nil, workerRequestTime: nil, @@ -417,5 +434,16 @@ func NewPrometheusMetrics(registry prometheus.Registerer) *PrometheusMetrics { panic(err) } + if err := m.registry.Register(m.opcacheRestarts); err != nil && + !errors.As(err, &prometheus.AlreadyRegisteredError{}) { + panic(err) + } + + // expose the series at zero so a rate or an alert on them works from the + // first restart on, instead of missing it for lack of a previous sample + for _, reason := range opcacheRestartReasons { + m.opcacheRestarts.WithLabelValues(reason) + } + return m } diff --git a/metrics_test.go b/metrics_test.go index 846a926569..99bbcfac4f 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -212,3 +212,30 @@ func TestPrometheusMetrics_TestStopReasonCrash(t *testing.T) { } } + +func TestPrometheusMetrics_OpcacheRestart(t *testing.T) { + m := NewPrometheusMetrics(prometheus.NewRegistry()) + m.OpcacheRestart("hash overflow") + m.OpcacheRestart("hash overflow") + m.OpcacheRestart("out of memory") + + // known reasons are exposed from the start, unknown ones only once seen + require.NoError(t, testutil.CollectAndCompare(m.opcacheRestarts, strings.NewReader(` + # HELP frankenphp_opcache_restarts Number of restarts of opcache's shared memory, by reason + # TYPE frankenphp_opcache_restarts counter + frankenphp_opcache_restarts{reason="hash overflow"} 2 + frankenphp_opcache_restarts{reason="out of memory"} 1 + frankenphp_opcache_restarts{reason="user"} 0 + `))) + + m.OpcacheRestart("unknown") + + require.NoError(t, testutil.CollectAndCompare(m.opcacheRestarts, strings.NewReader(` + # HELP frankenphp_opcache_restarts Number of restarts of opcache's shared memory, by reason + # TYPE frankenphp_opcache_restarts counter + frankenphp_opcache_restarts{reason="hash overflow"} 2 + frankenphp_opcache_restarts{reason="out of memory"} 1 + frankenphp_opcache_restarts{reason="unknown"} 1 + frankenphp_opcache_restarts{reason="user"} 0 + `))) +} From 81b29166ffe2fd8b912bbbb3efea8b829de864e2 Mon Sep 17 00:00:00 2001 From: Nicolas Grekas Date: Tue, 8 Sep 2026 10:15:37 +0200 Subject: [PATCH 3/3] docs: mark frankenphp_opcache_restarts as experimental Should always be zero, to be removed once opcache handles restarts safely under ZTS. --- docs/metrics.md | 2 +- metrics.go | 3 ++- metrics_test.go | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/metrics.md b/docs/metrics.md index 7a6a0981cf..c4bffb356e 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -23,7 +23,7 @@ When [Caddy metrics](https://caddyserver.com/docs/metrics) are enabled, FrankenP - `frankenphp_worker_crashes{worker="[worker_name]"}`: The number of times a worker has unexpectedly terminated. - `frankenphp_worker_restarts{worker="[worker_name]"}`: The number of times a worker has been deliberately restarted. - `frankenphp_worker_queue_depth{worker="[worker_name]"}`: The number of queued requests. -- `frankenphp_opcache_restarts{reason="[reason]"}`: The number of times opcache restarted its shared memory on its own, by reason (`out of memory`, `hash overflow`, `user`). Each restart is also logged. Under ZTS, running PHP threads may hold stale references to the rewound memory, so raise `opcache.memory_consumption` and `opcache.max_accelerated_files` when this counter grows. +- `frankenphp_opcache_restarts{reason="[reason]"}`: (experimental) The number of times opcache restarted its shared memory on its own, by reason (`out of memory`, `hash overflow`, `user`). This counter should always be zero: a restart rewinds memory that running PHP threads may still reference, which can crash the process. A non-zero value means opcache is undersized for the application, raise `opcache.memory_consumption` and `opcache.max_accelerated_files`. Each restart is also logged. This metric will be removed once opcache handles restarts safely under ZTS. For worker metrics, the `[worker_name]` placeholder is replaced by the worker name in the Caddyfile, otherwise the absolute path of the worker file will be used. diff --git a/metrics.go b/metrics.go index 5c9127649a..1f62d4aefe 100644 --- a/metrics.go +++ b/metrics.go @@ -405,9 +405,10 @@ func NewPrometheusMetrics(registry prometheus.Registerer) *PrometheusMetrics { Name: "frankenphp_queue_depth", Help: "Number of regular queued requests", }), + // experimental: to be removed once opcache handles restarts safely under ZTS opcacheRestarts: prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "frankenphp_opcache_restarts", - Help: "Number of restarts of opcache's shared memory, by reason", + Help: "Number of restarts of opcache's shared memory, by reason (experimental, should stay at zero)", }, []string{"reason"}), totalWorkers: nil, busyWorkers: nil, diff --git a/metrics_test.go b/metrics_test.go index 99bbcfac4f..132a0db0db 100644 --- a/metrics_test.go +++ b/metrics_test.go @@ -221,7 +221,7 @@ func TestPrometheusMetrics_OpcacheRestart(t *testing.T) { // known reasons are exposed from the start, unknown ones only once seen require.NoError(t, testutil.CollectAndCompare(m.opcacheRestarts, strings.NewReader(` - # HELP frankenphp_opcache_restarts Number of restarts of opcache's shared memory, by reason + # HELP frankenphp_opcache_restarts Number of restarts of opcache's shared memory, by reason (experimental, should stay at zero) # TYPE frankenphp_opcache_restarts counter frankenphp_opcache_restarts{reason="hash overflow"} 2 frankenphp_opcache_restarts{reason="out of memory"} 1 @@ -231,7 +231,7 @@ func TestPrometheusMetrics_OpcacheRestart(t *testing.T) { m.OpcacheRestart("unknown") require.NoError(t, testutil.CollectAndCompare(m.opcacheRestarts, strings.NewReader(` - # HELP frankenphp_opcache_restarts Number of restarts of opcache's shared memory, by reason + # HELP frankenphp_opcache_restarts Number of restarts of opcache's shared memory, by reason (experimental, should stay at zero) # TYPE frankenphp_opcache_restarts counter frankenphp_opcache_restarts{reason="hash overflow"} 2 frankenphp_opcache_restarts{reason="out of memory"} 1