From 511efe3da47b64a22272c5b7d8526349dfd3f9e9 Mon Sep 17 00:00:00 2001 From: mkn Date: Mon, 5 Oct 2026 14:07:34 +0200 Subject: [PATCH] feat(limits): raise PHP memory_limit automatically when a site runs out The pool sets memory_limit as php_admin_value, so a site that genuinely needs more (Elementor editor, WooCommerce imports) cannot raise it itself and just dies with "Allowed memory size ... exhausted" until someone edits the limits card. Opt-in per hosting on the PHP-FPM & DB limits card, with a user-set ceiling (default 512 MB). The owning node reads what nginx appended to the site's error.log every two minutes (O_NOFOLLOW, inode/offset tracked, history skipped on first sight) and, on an OOM at the limit in force, raises the pool 128 MB, bounded by the ceiling and by node RAM (limit x max_children <= 3/4 MemTotal), with a 10-minute cooldown. Two quiet weeks step it back down, never below the operator's own value; turning it off restores that value. Raises notify (info); a site still OOMing at its ceiling warns once a day. A hand-typed memory value becomes the new floor; saving the card unchanged does not adopt the raised value as the floor. Co-Authored-By: Claude Opus 5.5 --- bin/hyperion-agent/src/main.rs | 20 + bin/hyperion-web/src/handlers/hostings.rs | 104 ++++- .../templates/hostings_detail.html | 24 ++ crates/hyperion-adapters/src/fs.rs | 126 +++++- crates/hyperion-core/src/letters.rs | 56 +++ crates/hyperion-core/src/service.rs | 353 +++++++++++++++++ crates/hyperion-types/src/lib.rs | 1 + crates/hyperion-types/src/phpmem.rs | 362 ++++++++++++++++++ 8 files changed, 1044 insertions(+), 2 deletions(-) create mode 100644 crates/hyperion-types/src/phpmem.rs diff --git a/bin/hyperion-agent/src/main.rs b/bin/hyperion-agent/src/main.rs index d5a446b7..6831d9a5 100644 --- a/bin/hyperion-agent/src/main.rs +++ b/bin/hyperion-agent/src/main.rs @@ -1067,6 +1067,26 @@ async fn main() -> anyhow::Result<()> { } }); } + // Automatic PHP memory_limit: answer "Allowed memory size … exhausted" in + // a site's error.log by raising its pool one step (bounded), and step it + // back down after a quiet fortnight. Its own two-minute task: a site + // erroring out should not wait for the five-minute loop, and reading + // tenant logs must never hold fail2ban or quota enforcement up. + { + let mem_svc = svc.clone(); + tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_secs(90)).await; + let mut interval = tokio::time::interval(std::time::Duration::from_secs(120)); + loop { + interval.tick().await; + match mem_svc.php_mem_auto_tick().await { + Ok(n) if n > 0 => tracing::info!(pools = n, "php mem auto: rewrote pools"), + Ok(_) => {} + Err(e) => tracing::warn!(error=%e, "php mem auto tick failed"), + } + } + }); + } // The weekly page walk. On its OWN task, not in the five-minute loop: // one pass fetches up to eight pages plus forty links PER SITE, each // with a 20-second ceiling, so on a node with a few dozen care-plan diff --git a/bin/hyperion-web/src/handlers/hostings.rs b/bin/hyperion-web/src/handlers/hostings.rs index 10bf278b..c30f4cf6 100644 --- a/bin/hyperion-web/src/handlers/hostings.rs +++ b/bin/hyperion-web/src/handlers/hostings.rs @@ -126,6 +126,7 @@ struct DetailTpl<'a> { htmx_version: &'static str, detail: HostingDetail, limits: hyperion_types::HostingLimits, + mem_auto: PhpMemAutoView, wp_status: Option, expiry: hyperion_types::HostingExpiry, backups: Vec, @@ -1394,6 +1395,7 @@ pub async fn post_create( .unwrap_or_else(|_| hyperion_types::HostingLimits::defaults()); let staging_domain_default = format!("staging.{}", detail.domain); let preview_domain = compute_preview_domain(&state, target, &detail.domain).await; + let mem_auto = PhpMemAutoView::from_kv(&[], limits.php_memory_mb); let tpl = DetailTpl { username: &ctx.username, user_initial: super::user_initial(&ctx.username), @@ -1402,6 +1404,7 @@ pub async fn post_create( htmx_version: super::htmx_version(), detail, limits, + mem_auto, wp_status: None, expiry: hyperion_types::HostingExpiry::defaults(), backups: vec![], @@ -1788,6 +1791,43 @@ pub(crate) fn compute_hosting_health( } } +/// The automatic PHP memory_limit, as the OWNING node stores it +/// (`hyperion_types::phpmem`). +pub struct PhpMemAutoView { + pub enabled: bool, + /// The most the automation may set, in MiB. + pub max_mb: i64, + /// The operator's own limit, when the automation currently has the pool + /// above it. + pub raised_from_mb: Option, + /// Last out-of-memory fatal seen; 0 = none since it was turned on. + pub last_oom_at: i64, +} + +impl PhpMemAutoView { + fn from_kv(pairs: &[(String, String)], current_mb: i64) -> Self { + use hyperion_types::phpmem; + let get = |k: &str| { + pairs + .iter() + .find(|(key, _)| key == k) + .map(|(_, v)| v.as_str()) + }; + let st = get(phpmem::KV_STATE).and_then(phpmem::State::parse); + PhpMemAutoView { + enabled: get(phpmem::KV_ENABLED) == Some("on"), + max_mb: get(phpmem::KV_MAX_MB) + .and_then(|v| v.trim().parse().ok()) + .unwrap_or(phpmem::DEFAULT_MAX_MB), + raised_from_mb: st + .as_ref() + .map(|s| s.base_mb) + .filter(|&b| b > 0 && b < current_mb), + last_oom_at: st.map(|s| s.last_oom_at).unwrap_or(0), + } + } +} + /// Operator-tunable php.ini settings, applied via a per-hosting /// `.user.ini` in htdocs (PHP_INI_PERDIR — no FPM pool change, so a bad /// value can never stop the pool from starting). Empty string = leave @@ -2213,6 +2253,7 @@ pub async fn get_detail( Ok(RpcResponse::HostingKvList(v)) => v, _ => vec![], }; + let mem_auto = PhpMemAutoView::from_kv(&owner_kv, limits.php_memory_mb); let backup_cadence = owner_kv .iter() .find(|(k, _)| k == "backup_cadence") @@ -2398,6 +2439,7 @@ pub async fn get_detail( htmx_version: super::htmx_version(), detail, limits, + mem_auto, wp_status, expiry, backups, @@ -4521,6 +4563,11 @@ pub struct SetLimitsForm { php_max_children: i64, php_max_requests: i64, db_max_connections: i64, + /// Checkbox: absent when unticked. + #[serde(default)] + mem_auto: String, + #[serde(default)] + mem_auto_max_mb: String, #[serde(default)] target_node: String, } @@ -5007,13 +5054,68 @@ pub async fn post_set_limits( // The enforced caps are the hosting_quotas row (the quota card), which // profiles also write through quota_set. Saving this card now clears // any stale ghost value, so old rows converge on the truth. + let mem_auto_max: i64 = match form.mem_auto_max_mb.trim() { + "" => hyperion_types::phpmem::DEFAULT_MAX_MB, + v => match v.parse::() { + Ok(n) if (16..=8192).contains(&n) => n, + _ => { + return Ok(save_result( + &headers, + false, + "Automatic memory ceiling must be 16–8192 MB.", + format!( + "/hostings/{}?flash_error={}#limits", + urlencoding(&form.selector), + urlencoding("Automatic memory ceiling must be 16–8192 MB.") + ), + )) + } + }, + }; + let mem_auto_on = form.mem_auto.trim() == "on"; let target = node_target(&form.target_node); let resp = crate::dispatcher::dispatch_to_node( &state, target, - Request::HostingSetLimits { sel, limits: l }, + Request::HostingSetLimits { + sel: sel.clone(), + limits: l, + }, ) .await?; + if let RpcResponse::HostingSetLimits(_) = resp { + // The automation's switch and ceiling live in the OWNING node's + // hosting_kv, beside the state its tick keeps — the pool it rewrites + // is there. Written after the limits, so a refused save above leaves + // them as they were. + let hosting_id = match find_hosting_anywhere(&state, sel).await { + Ok((d, _)) => d.id.as_str().to_string(), + Err(e) => return Err(e), + }; + for (key, value) in [ + ( + hyperion_types::phpmem::KV_ENABLED, + if mem_auto_on { "on" } else { "off" }.to_string(), + ), + (hyperion_types::phpmem::KV_MAX_MB, mem_auto_max.to_string()), + ] { + match crate::dispatcher::dispatch_to_node( + &state, + target, + Request::HostingKvSet { + hosting_id: hosting_id.clone(), + key: key.into(), + value, + }, + ) + .await? + { + RpcResponse::HostingKvSet => {} + RpcResponse::Error(e) => return Err(AppError::Rpc(e.to_string())), + _ => return Err(AppError::Internal("unexpected response".into())), + } + } + } match resp { RpcResponse::HostingSetLimits(_) => Ok(save_result( &headers, diff --git a/bin/hyperion-web/templates/hostings_detail.html b/bin/hyperion-web/templates/hostings_detail.html index e657850a..1d9f87fd 100644 --- a/bin/hyperion-web/templates/hostings_detail.html +++ b/bin/hyperion-web/templates/hostings_detail.html @@ -2763,6 +2763,30 @@

value="{{ limits.db_max_connections }}"> + {# Automatic memory_limit: the owning node watches this site's error.log + for "Allowed memory size … exhausted" and raises the pool one 128 MB + step at a time up to the ceiling (and what the node's RAM allows); + two quiet weeks take it one step back. Unticking restores the value + above. State lives in the owning node's hosting_kv. #} +
+
+ +
+
+ + +
+
+ {% if mem_auto.enabled %} +

+ {% if let Some(from) = mem_auto.raised_from_mb %}Raised automatically from your {{ from }} MB to {{ limits.php_memory_mb }} MB.{% else %}At your own value.{% endif %} + {% if mem_auto.last_oom_at > 0 %}Last ran out of memory {{ crate::handlers::stats::fmt_ago(mem_auto.last_oom_at) }}.{% else %}No out-of-memory errors seen since it was turned on.{% endif %} +

+ {% endif %} {# Disk and bandwidth caps are deliberately NOT here. This card once carried its own "Disk limit" and "Monthly bandwidth" fields, and they were placebo: stored in a column nothing enforced and nothing alerted diff --git a/crates/hyperion-adapters/src/fs.rs b/crates/hyperion-adapters/src/fs.rs index 14f555f8..7b53aabf 100644 --- a/crates/hyperion-adapters/src/fs.rs +++ b/crates/hyperion-adapters/src/fs.rs @@ -81,9 +81,12 @@ pub fn open_no_follow(path: &Path) -> Result { use rustix::fs::{Mode, OFlags}; use std::os::unix::fs::MetadataExt; + // NONBLOCK: a FIFO in place of the log would otherwise block the open + // itself — before the regular-file check below could refuse it — and + // hang whichever loop asked. It changes nothing for a regular file. let fd = rustix::fs::open( path, - OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK, Mode::empty(), ) .map_err(|e| { @@ -196,6 +199,75 @@ pub fn truncate_no_follow(path: &Path) -> Result<(), AdapterError> { /// /// Replaces shelling out to `tail`, which takes a PATH and therefore resolves /// it again — after any check the caller made, and as root. +/// What [`read_appended`] found: the new text plus where to resume. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Appended { + pub text: String, + pub ino: u64, + pub offset: u64, +} + +/// Read what was appended to a log since `(ino, offset)`, the way `tail -F` +/// would, opened through [`open_no_follow`] (the logs live in a directory +/// the site user owns). +/// +/// * First sight (`ino == 0`): nothing is returned and the position is set +/// to the end — history is not news. +/// * Rotated (inode changed) or truncated (shorter than `offset`): read the +/// new file from its start. +/// * More than `max` bytes new: only the newest `max` are read. +/// +/// The returned offset stops after the last complete line, so a line nginx +/// is still writing is read whole next time instead of half now. +pub fn read_appended( + path: &Path, + ino: u64, + offset: u64, + max: u64, +) -> Result { + use std::io::{Read, Seek, SeekFrom}; + use std::os::unix::fs::MetadataExt; + + let mut file = open_no_follow(path)?; + let md = file + .metadata() + .map_err(|e| AdapterError::Other(format!("stat: {e}")))?; + let (cur_ino, len) = (md.ino(), md.len()); + if ino == 0 { + return Ok(Appended { + text: String::new(), + ino: cur_ino, + offset: len, + }); + } + let from = if cur_ino != ino || len < offset { + 0 + } else { + offset + }; + let start = from.max(len.saturating_sub(max)); + if start >= len { + return Ok(Appended { + text: String::new(), + ino: cur_ino, + offset: len, + }); + } + file.seek(SeekFrom::Start(start)) + .map_err(|e| AdapterError::Other(format!("seek: {e}")))?; + let mut buf = Vec::with_capacity((len - start) as usize); + file.take(len - start) + .read_to_end(&mut buf) + .map_err(|e| AdapterError::Other(format!("read: {e}")))?; + let complete = buf.iter().rposition(|&b| b == b'\n').map_or(0, |i| i + 1); + buf.truncate(complete); + Ok(Appended { + text: String::from_utf8_lossy(&buf).into_owned(), + ino: cur_ino, + offset: start + complete as u64, + }) +} + pub async fn tail_lines(path: &Path, lines: usize) -> Result { use std::io::{Read, Seek, SeekFrom}; @@ -449,6 +521,58 @@ fn with_extension(p: &Path, ext: &str) -> PathBuf { mod tests { use super::*; + #[test] + fn read_appended_follows_a_growing_log() { + use std::io::Write; + let d = tempfile::tempdir().expect("tempdir"); + let p = d.path().join("error.log"); + std::fs::write(&p, "old line\n").expect("write"); + + // First sight: history is skipped. + let a = read_appended(&p, 0, 0, 1 << 20).expect("read"); + assert_eq!(a.text, ""); + assert_eq!(a.offset, 9); + + // Appended text, with a half-written last line held back. + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(&p) + .expect("open"); + f.write_all(b"one\ntwo\npart").expect("append"); + let b = read_appended(&p, a.ino, a.offset, 1 << 20).expect("read"); + assert_eq!(b.text, "one\ntwo\n"); + assert_eq!(b.offset, 17); + + // The finished line arrives next time. + f.write_all(b"ial\n").expect("append"); + let c = read_appended(&p, b.ino, b.offset, 1 << 20).expect("read"); + assert_eq!(c.text, "partial\n"); + + // Truncated (rotation by copytruncate): read from the start. + std::fs::write(&p, "fresh\n").expect("write"); + let e = read_appended(&p, c.ino, c.offset, 1 << 20).expect("read"); + assert_eq!(e.text, "fresh\n"); + + // A burst larger than `max`: only the newest part. + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(&p) + .expect("open"); + f.write_all(b"aaaaaaaaaa\nbbbb\n").expect("append"); + let g = read_appended(&p, e.ino, e.offset, 8).expect("read"); + assert_eq!(g.text, "aa\nbbbb\n"); + } + + #[test] + fn read_appended_refuses_a_symlink() { + let d = tempfile::tempdir().expect("tempdir"); + let target = d.path().join("secret"); + std::fs::write(&target, "x\n").expect("write"); + let link = d.path().join("error.log"); + std::os::unix::fs::symlink(&target, &link).expect("symlink"); + assert!(read_appended(&link, 1, 0, 1 << 20).is_err()); + } + #[tokio::test] async fn atomic_write_creates_parent_and_file() { let d = tempfile::tempdir().expect("tempdir"); diff --git a/crates/hyperion-core/src/letters.rs b/crates/hyperion-core/src/letters.rs index 4969f9a5..a79c503b 100644 --- a/crates/hyperion-core/src/letters.rs +++ b/crates/hyperion-core/src/letters.rs @@ -1124,6 +1124,62 @@ pub static STRINGS: &[LetterString] = &[ en: "{domain} — the automatic page check found {count} page(s) that do not work: {urls}", cs: "{domain} — automatická kontrola stránek našla {count} nefunkčních stránek: {urls}", }, + LetterString { + id: "ops.php_mem_raised.title", + group: "Operator alerts — hostings", + note: "The automatic memory_limit raised a site's PHP memory.", + en: "PHP memory raised for a site", + cs: "Webu byla navýšena paměť PHP", + }, + LetterString { + id: "ops.php_mem_raised.body", + group: "Operator alerts — hostings", + note: "Tokens: {domain} {from} {to} {ceiling} (MiB).", + en: "{domain} ran out of PHP memory at {from} MB, so Hyperion raised its memory_limit \ + to {to} MB (ceiling {ceiling} MB). It steps back down after two weeks without \ + running out.", + cs: "{domain} došla paměť PHP při {from} MB, takže Hyperion zvýšil memory_limit na \ + {to} MB (strop {ceiling} MB). Po dvou týdnech bez nedostatku paměti se sníží \ + zpět.", + }, + LetterString { + id: "ops.php_mem_capped.title", + group: "Operator alerts — hostings", + note: "A site keeps running out of PHP memory at its automatic ceiling.", + en: "Site still runs out of PHP memory at its ceiling", + cs: "Webu dochází paměť PHP i na stropu", + }, + LetterString { + id: "ops.php_mem_capped.body", + group: "Operator alerts — hostings", + note: "Tokens: {domain} {limit} (MiB).", + en: "{domain} keeps running out of PHP memory at {limit} MB, the ceiling set for it. \ + That much is rarely a real need: look for a plugin or theme leaking memory, or \ + raise the ceiling on the site's PHP-FPM & DB limits card if it is deliberate.", + cs: "{domain} dochází paměť PHP i při {limit} MB, což je jeho nastavený strop. Tolik \ + paměti je skutečně potřeba jen výjimečně: hledejte plugin nebo šablonu, která \ + paměť plýtvá, nebo strop vědomě zvyšte v kartě PHP-FPM & DB limity.", + }, + LetterString { + id: "ops.php_mem_capped_ram.title", + group: "Operator alerts — hostings", + note: "A site runs out of PHP memory but the node's RAM stops a further raise.", + en: "Site runs out of PHP memory; server RAM blocks a raise", + cs: "Webu dochází paměť PHP; RAM serveru brání navýšení", + }, + LetterString { + id: "ops.php_mem_capped_ram.body", + group: "Operator alerts — hostings", + note: "Tokens: {domain} {limit} (MiB).", + en: "{domain} keeps running out of PHP memory at {limit} MB. Its ceiling allows more, \ + but every PHP worker the site may run at once would then not fit in three \ + quarters of this server's RAM. Lower the site's max_children, add RAM, or find \ + what is using the memory.", + cs: "{domain} dochází paměť PHP při {limit} MB. Strop by dovolil víc, ale všechny \ + PHP procesy, které web smí spustit najednou, by se pak nevešly do tří čtvrtin \ + RAM serveru. Snižte webu max_children, přidejte RAM, nebo zjistěte, co paměť \ + spotřebovává.", + }, LetterString { id: "ops.mail_override.title", group: "Operator alerts — WordPress", diff --git a/crates/hyperion-core/src/service.rs b/crates/hyperion-core/src/service.rs index 782b78b3..353f1d2d 100644 --- a/crates/hyperion-core/src/service.rs +++ b/crates/hyperion-core/src/service.rs @@ -4264,6 +4264,31 @@ impl HostingService { limits: hyperion_types::HostingLimits, ) -> Result { let detail = self.get(sel).await?; + let before = hyperion_state::limits::get(&self.pool, &detail.id) + .await + .ok() + .flatten() + .map(|r| r.php_memory_mb); + let limits = self.store_and_apply_limits(&detail, limits).await?; + // A memory value typed by hand (or applied by a profile) is the new + // floor for the automatic memory_limit. Saving the card UNCHANGED — + // it shows the value the automation raised to — must not adopt that + // raised value as the operator's own, or it would never come down. + if before != Some(limits.php_memory_mb) { + self.php_mem_auto_rebase(detail.id.as_str(), limits.php_memory_mb) + .await; + } + Ok(limits) + } + + /// Persist the limits row and rewrite the pool, without touching the + /// automatic-memory floor. Shared by the operator path above and by + /// [`Self::php_mem_auto_tick`]. + async fn store_and_apply_limits( + &self, + detail: &HostingDetail, + limits: hyperion_types::HostingLimits, + ) -> Result { let limits = clamp_limits(limits); let row = limits_to_row(&detail.id, &limits, now_secs()); hyperion_state::limits::upsert(&self.pool, &row) @@ -4287,6 +4312,226 @@ impl HostingService { Ok(limits) } + /// Make `base_mb` the floor of an existing automatic-memory state. + async fn php_mem_auto_rebase(&self, hosting_id: &str, base_mb: i64) { + use hyperion_types::phpmem; + let Some(mut st) = + hyperion_state::hosting_kv::get(&self.pool, hosting_id, phpmem::KV_STATE) + .await + .ok() + .flatten() + .and_then(|v| phpmem::State::parse(&v)) + else { + return; + }; + let now = now_secs(); + st.base_mb = base_mb; + st.last_change_at = now; + let _ = hyperion_state::hosting_kv::set( + &self.pool, + hosting_id, + phpmem::KV_STATE, + &st.to_json(), + now, + ) + .await; + } + + /// Automatic PHP `memory_limit` (see `hyperion_types::phpmem`). + /// + /// For every live site with the automation on, read what nginx appended + /// to its `error.log`, and when PHP ran out of memory at the limit in + /// force, raise the pool's limit one step — bounded by the site's ceiling + /// and by the node's RAM. A fortnight without running out takes it one + /// step back towards the operator's own value. Turning the automation off + /// puts the operator's value back. + /// + /// Returns how many pools it rewrote. + pub async fn php_mem_auto_tick(&self) -> Result { + use hyperion_types::phpmem::{self, Step}; + let mem_total_mb = read_mem_total_mb().await; + let mut acted = 0i64; + for s in self.list().await? { + let id = s.id.as_str().to_string(); + let kv = |k: &'static str| { + let id = id.clone(); + async move { + hyperion_state::hosting_kv::get(&self.pool, &id, k) + .await + .ok() + .flatten() + } + }; + let enabled = kv(phpmem::KV_ENABLED).await.as_deref() == Some("on"); + let stored = kv(phpmem::KV_STATE) + .await + .and_then(|v| phpmem::State::parse(&v)); + if !enabled && stored.is_none() { + continue; + } + // Suspended or trashed: the pool is not serving, nothing to learn + // from its log and nothing worth rewriting. + if s.state != HostingState::Active { + continue; + } + let Ok(detail) = self.get(HostingSelector::Id(s.id.clone())).await else { + continue; + }; + if detail.php_version.is_none() || detail.system_user.trim().is_empty() { + continue; + } + let limits = hyperion_state::limits::get(&self.pool, &detail.id) + .await + .ok() + .flatten() + .map(row_to_limits) + .unwrap_or_else(hyperion_types::HostingLimits::defaults); + let current = limits.php_memory_mb; + let now = now_secs(); + + if !enabled { + // Turned off: hand back the operator's value, then forget. + let base = stored.map(|st| st.base_mb).unwrap_or(current); + if base > 0 && base != current { + let mut l = limits.clone(); + l.php_memory_mb = base; + if let Err(e) = self.store_and_apply_limits(&detail, l).await { + tracing::warn!(domain = %detail.domain, error = %e, "php mem auto: restoring the operator's limit failed; will retry"); + continue; + } + acted += 1; + self.append_audit( + "php.mem_auto.restore", + Some(&id), + &serde_json::json!({"from_mb": current, "to_mb": base}).to_string(), + "ok", + ) + .await; + } + let _ = hyperion_state::hosting_kv::delete(&self.pool, &id, phpmem::KV_STATE).await; + continue; + } + + let mut st = stored.unwrap_or_else(|| phpmem::State { + base_mb: current, + ..Default::default() + }); + let before = st.clone(); + let path = std::path::PathBuf::from(&self.paths.home_root) + .join(&detail.system_user) + .join(&detail.domain) + .join("logs") + .join("error.log"); + let (ino, off) = (st.log_ino, st.log_offset); + let read = tokio::task::spawn_blocking(move || { + hyperion_adapters::fs::read_appended(&path, ino, off, phpmem::MAX_SCAN_BYTES) + }) + .await; + match read { + Ok(Ok(a)) => { + if phpmem::count_ooms(&a.text, current) > 0 { + st.last_oom_at = now; + } + st.log_ino = a.ino; + st.log_offset = a.offset; + } + // No log yet, or one root must not read (a symlink): nothing + // seen. The lowering below still runs on time alone. + Ok(Err(e)) => { + tracing::debug!(domain = %detail.domain, error = %e, "php mem auto: error.log not read") + } + Err(e) => tracing::warn!(error = %e, "php mem auto: log reader panicked"), + } + + let ceiling = kv(phpmem::KV_MAX_MB) + .await + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(phpmem::DEFAULT_MAX_MB) + .clamp(16, 8192); + let ram_cap = mem_total_mb.and_then(|t| phpmem::ram_cap_mb(t, limits.php_max_children)); + let href = format!("/hostings/{}#limits", detail.domain); + match phpmem::decide(now, current, ceiling, ram_cap, &st) { + Step::Nothing => {} + Step::Raise { to } | Step::Lower { to } if to == current => {} + step @ (Step::Raise { to } | Step::Lower { to }) => { + let raise = matches!(step, Step::Raise { .. }); + let mut l = limits.clone(); + l.php_memory_mb = to; + match self.store_and_apply_limits(&detail, l).await { + Ok(_) => { + st.last_change_at = now; + acted += 1; + let (from_s, to_s) = (current.to_string(), to.to_string()); + self.append_audit( + if raise { + "php.mem_auto.raise" + } else { + "php.mem_auto.lower" + }, + Some(&id), + &serde_json::json!({"from_mb": current, "to_mb": to, "ceiling_mb": ceiling}).to_string(), + "ok", + ) + .await; + if raise { + let ceiling_s = ceiling.to_string(); + self.notify_admins_say( + "info", + "ops.php_mem_raised", + &[ + ("domain", detail.domain.as_str()), + ("from", from_s.as_str()), + ("to", to_s.as_str()), + ("ceiling", ceiling_s.as_str()), + ], + &href, + "php_mem_auto", + ) + .await; + } + } + Err(e) => { + // State untouched (bar the log position), so the + // next tick tries the same step again. + tracing::warn!(domain = %detail.domain, to_mb = to, error = %e, "php mem auto: pool rewrite failed; will retry"); + } + } + } + Step::Capped { at, by_ram } => { + st.capped_alert_at = now; + let at_s = at.to_string(); + // Still running out at the most it may get. Not an + // outage of the whole site — the pages that need less + // still serve — so a warning: a plugin to look at, or a + // ceiling to raise on purpose. + self.notify_admins_say( + "warn", + if by_ram { + "ops.php_mem_capped_ram" + } else { + "ops.php_mem_capped" + }, + &[("domain", detail.domain.as_str()), ("limit", at_s.as_str())], + &href, + "php_mem_auto", + ) + .await; + } + } + if st != before { + let _ = hyperion_state::hosting_kv::set( + &self.pool, + &id, + phpmem::KV_STATE, + &st.to_json(), + now, + ) + .await; + } + } + Ok(acted) + } + pub async fn get_limits( &self, sel: HostingSelector, @@ -33919,6 +34164,18 @@ async fn read_psi_some_avg10(resource: &str) -> i64 { 0 } +/// `MemTotal` from /proc/meminfo, in MiB. `None` off Linux or on a parse +/// failure — callers then skip the RAM bound rather than guess one. +async fn read_mem_total_mb() -> Option { + let s = tokio::fs::read_to_string("/proc/meminfo").await.ok()?; + s.lines() + .find_map(|l| l.strip_prefix("MemTotal:")) + .and_then(|rest| rest.split_whitespace().next()) + .and_then(|kib| kib.parse::().ok()) + .map(|kib| kib / 1024) + .filter(|&mb| mb > 0) +} + async fn read_proc_metrics() -> ProcMetrics { let loadavg = tokio::fs::read_to_string("/proc/loadavg").await.ok(); let la_1m = loadavg @@ -42618,6 +42875,102 @@ mod tests { assert!(susp.is_none(), "suspension row removed on resume"); } + /// Out-of-memory in the site's error.log raises the pool one step; a hand + /// save of the UNCHANGED card keeps the operator's floor, a changed value + /// becomes the new floor; turning the automation off restores the floor. + #[tokio::test] + async fn php_mem_auto_raises_rebases_and_restores() { + use hyperion_types::phpmem; + use std::io::Write; + let pool = open_memory().await.expect("open"); + let home = tempfile::tempdir().expect("dir"); + let mut a = happy_mocks(); + a.expect_apply_php_limits() + .returning(|_, _, _, _, _, _, _| Ok(())); + let s = svc(pool.clone(), a).with_paths(HostingPaths { + home_root: home.path().display().to_string(), + ..HostingPaths::default() + }); + s.create(req("ex.cz")).await.expect("create"); + let sel = HostingSelector::Domain(Domain::parse("ex.cz").unwrap()); + let d = s.get(sel.clone()).await.expect("get"); + let id = d.id.as_str().to_string(); + let logs = home + .path() + .join(&d.system_user) + .join(&d.domain) + .join("logs"); + std::fs::create_dir_all(&logs).expect("mkdir"); + let log = logs.join("error.log"); + std::fs::write( + &log, + "Allowed memory size of 268435456 bytes exhausted (old)\n", + ) + .expect("write"); + hyperion_state::hosting_kv::set(&pool, &id, phpmem::KV_ENABLED, "on", 0) + .await + .expect("kv"); + let state = || async { + phpmem::State::parse( + &hyperion_state::hosting_kv::get(&pool, &id, phpmem::KV_STATE) + .await + .expect("kv") + .unwrap_or_default(), + ) + }; + let mem = || async { + s.get_limits(sel.clone()) + .await + .expect("limits") + .php_memory_mb + }; + + // First sight: the history already in the log is not acted on. + assert_eq!(s.php_mem_auto_tick().await.expect("tick"), 0); + assert_eq!(mem().await, 256); + assert_eq!(state().await.expect("state").base_mb, 256); + + let mut f = std::fs::OpenOptions::new() + .append(true) + .open(&log) + .expect("open"); + f.write_all( + b"FastCGI sent in stderr: \"PHP message: PHP Fatal error: Allowed memory size of 268435456 bytes exhausted (tried to allocate 20480 bytes)\"\n", + ) + .expect("append"); + assert_eq!(s.php_mem_auto_tick().await.expect("tick"), 1); + assert_eq!(mem().await, 384); + // Same lines are not read twice, and the cooldown holds anyway. + assert_eq!(s.php_mem_auto_tick().await.expect("tick"), 0); + + // Saving the card as shown keeps the operator's own 256 as the floor. + let mut l = s.get_limits(sel.clone()).await.expect("limits"); + s.set_limits(sel.clone(), l.clone()).await.expect("set"); + assert_eq!(state().await.expect("state").base_mb, 256); + // A new value typed by hand becomes the floor. + l.php_memory_mb = 320; + s.set_limits(sel.clone(), l).await.expect("set"); + assert_eq!(state().await.expect("state").base_mb, 320); + + // Raise once more, then switch off: the floor comes back. + f.write_all(b"Allowed memory size of 335544320 bytes exhausted\n") + .expect("append"); + let mut st = state().await.expect("state"); + st.last_change_at = 0; // past the cooldown + hyperion_state::hosting_kv::set(&pool, &id, phpmem::KV_STATE, &st.to_json(), 0) + .await + .expect("kv"); + assert_eq!(s.php_mem_auto_tick().await.expect("tick"), 1); + assert_eq!(mem().await, 448); + hyperion_state::hosting_kv::set(&pool, &id, phpmem::KV_ENABLED, "off", 0) + .await + .expect("kv"); + assert_eq!(s.php_mem_auto_tick().await.expect("tick"), 1); + assert_eq!(mem().await, 320); + assert!(state().await.is_none(), "state forgotten once off"); + assert_eq!(s.php_mem_auto_tick().await.expect("tick"), 0); + } + #[tokio::test] async fn set_limits_clamps_and_persists() { let pool = open_memory().await.expect("open"); diff --git a/crates/hyperion-types/src/lib.rs b/crates/hyperion-types/src/lib.rs index 4b05e4bd..5ff77ef4 100644 --- a/crates/hyperion-types/src/lib.rs +++ b/crates/hyperion-types/src/lib.rs @@ -29,6 +29,7 @@ pub mod migration; pub mod package; pub mod perf; pub mod php; +pub mod phpmem; pub mod profile; pub mod regguard; pub mod report_sections; diff --git a/crates/hyperion-types/src/phpmem.rs b/crates/hyperion-types/src/phpmem.rs new file mode 100644 index 00000000..1108e790 --- /dev/null +++ b/crates/hyperion-types/src/phpmem.rs @@ -0,0 +1,362 @@ +//! Automatic PHP `memory_limit` — raise it when a site runs out, lower it +//! again once the site has been quiet for a while. +//! +//! The pool sets `memory_limit` as `php_admin_value`, so nothing inside the +//! site (`WP_MEMORY_LIMIT`, `ini_set`) can raise it; a site that genuinely +//! needs more — an Elementor editor, a WooCommerce import — just dies with +//! `Allowed memory size of … bytes exhausted` until someone edits the limits +//! card. PHP writes that fatal to FastCGI stderr, which nginx appends to the +//! site's own `logs/error.log`, so the owning node can see it without the +//! site's help and act on it. +//! +//! Everything here is pure: the decision, the log-line parser and the stored +//! state. The node does the I/O. All of it lives in the OWNING node's +//! `hosting_kv`, because the pool it rewrites is there. +//! +//! Bounds, because "dynamic" must never mean "unbounded": +//! * never above the operator's ceiling for the site (`KV_MAX_MB`); +//! * never above what the node can hold — `limit × max_children` stays under +//! three quarters of the node's RAM; +//! * never below the operator's own value (`State::base_mb`) when lowering. + +use serde::{Deserialize, Serialize}; + +/// `"on"` turns the automation on for a hosting. Anything else is off. +pub const KV_ENABLED: &str = "php.mem_auto"; +/// The highest `memory_limit` (MiB) the automation may set for a hosting. +pub const KV_MAX_MB: &str = "php.mem_auto_max_mb"; +/// JSON-encoded [`State`]. +pub const KV_STATE: &str = "php.mem_auto_state"; + +/// Ceiling when the operator has not set one. Covers Elementor and +/// WooCommerce; a site that needs more than this almost always has a plugin +/// leaking, and raising further would only hide it. +pub const DEFAULT_MAX_MB: i64 = 512; +/// How much one raise or one lowering moves the limit. +pub const STEP_MB: i64 = 128; +/// Minimum gap between two changes, so one burst of fatals is answered with +/// one step and the next step is decided on what the NEW limit does. +pub const COOLDOWN_SECS: i64 = 10 * 60; +/// Quiet period (no out-of-memory, no change) before one step back down. +pub const LOWER_AFTER_SECS: i64 = 14 * 86_400; +/// How often a site stuck at its ceiling is reported again. +pub const CAPPED_ALERT_EVERY_SECS: i64 = 86_400; +/// Most of `error.log` read per tick. A site logging more than this between +/// two ticks is read from the newest part only. +pub const MAX_SCAN_BYTES: u64 = 8 * 1024 * 1024; + +/// What the node remembers per hosting between ticks. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct State { + /// The operator's own `memory_limit` — the floor a lowering returns to. + /// Reset whenever someone sets the limit by hand. + pub base_mb: i64, + /// Last out-of-memory fatal seen AT OR ABOVE the limit in force. + #[serde(default)] + pub last_oom_at: i64, + /// Last time the limit changed (by this automation or by hand). + #[serde(default)] + pub last_change_at: i64, + /// Last "stuck at the ceiling" alert. + #[serde(default)] + pub capped_alert_at: i64, + /// Inode of `error.log` when last read, to notice rotation. + #[serde(default)] + pub log_ino: u64, + /// Byte offset read up to. + #[serde(default)] + pub log_offset: u64, +} + +impl State { + pub fn parse(s: &str) -> Option { + serde_json::from_str(s).ok() + } + + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + + /// An out-of-memory seen since the last change and not yet answered. + /// + /// Remembered rather than taken from this tick's lines alone: fatals that + /// land during the cooldown are read (the offset moves past them) and + /// would otherwise be forgotten by the time the cooldown ends. + pub fn oom_pending(&self) -> bool { + self.last_oom_at > self.last_change_at + } +} + +/// What the tick should do for one hosting. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Step { + Nothing, + /// Rewrite the pool with this `memory_limit` (MiB). + Raise { + to: i64, + }, + Lower { + to: i64, + }, + /// Still running out, but already at the most it may get. `by_ram` when + /// the node's memory, not the operator's ceiling, is what stops it. + Capped { + at: i64, + by_ram: bool, + }, +} + +/// The `N` in `Allowed memory size of N bytes exhausted`, from one log line. +pub fn parse_oom_limit_bytes(line: &str) -> Option { + const NEEDLE: &str = "Allowed memory size of "; + let rest = &line[line.find(NEEDLE)? + NEEDLE.len()..]; + let digits_end = rest + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(rest.len()); + if digits_end == 0 || !rest[digits_end..].starts_with(" bytes exhausted") { + return None; + } + rest[..digits_end].parse().ok() +} + +/// Out-of-memory fatals in `text` that hit a limit of at least +/// `current_mb`. +/// +/// A fatal at a LOWER limit is from before the last raise — a line nginx +/// wrote after the tick read past the change, or an old log — and must not +/// push the limit up again. +pub fn count_ooms(text: &str, current_mb: i64) -> usize { + let floor = (current_mb.max(0) as u64) * 1024 * 1024; + text.lines() + .filter_map(parse_oom_limit_bytes) + .filter(|&n| n >= floor) + .count() +} + +/// Decide one hosting's step. `st.last_oom_at` must already include this +/// tick's fatals. `ram_cap_mb` is the most the node can afford per worker +/// (`None` when the node's memory could not be read). +pub fn decide( + now: i64, + current_mb: i64, + ceiling_mb: i64, + ram_cap_mb: Option, + st: &State, +) -> Step { + let base = st.base_mb; + // The operator lowered the ceiling under a value this automation set. + // Never below their own base: that one is theirs, not ours. + let lower_bound = ceiling_mb.max(base); + if current_mb > lower_bound { + return Step::Lower { to: lower_bound }; + } + + if st.oom_pending() { + let ram = ram_cap_mb.unwrap_or(i64::MAX); + let cap = ceiling_mb.min(ram); + if current_mb >= cap { + if st.capped_alert_at > 0 && now - st.capped_alert_at < CAPPED_ALERT_EVERY_SECS { + return Step::Nothing; + } + return Step::Capped { + at: current_mb, + by_ram: ram < ceiling_mb, + }; + } + if now - st.last_change_at < COOLDOWN_SECS { + return Step::Nothing; + } + return Step::Raise { + to: (current_mb + STEP_MB).min(cap), + }; + } + + let quiet_since = st.last_oom_at.max(st.last_change_at); + if current_mb > base && now - quiet_since >= LOWER_AFTER_SECS { + return Step::Lower { + to: (current_mb - STEP_MB).max(base), + }; + } + Step::Nothing +} + +/// Per-worker memory the node can afford: three quarters of its RAM shared +/// by every worker the pool may run at once. +pub fn ram_cap_mb(mem_total_mb: i64, max_children: i64) -> Option { + if mem_total_mb <= 0 { + return None; + } + Some(mem_total_mb * 3 / 4 / max_children.max(1)) +} + +#[cfg(test)] +mod tests { + use super::*; + + const MIB: u64 = 1024 * 1024; + + fn st(base: i64) -> State { + State { + base_mb: base, + ..State::default() + } + } + + #[test] + fn parses_the_nginx_fastcgi_line() { + let l = r#"2026/10/05 12:00:01 [error] 123#123: *45 FastCGI sent in stderr: "PHP message: PHP Fatal error: Allowed memory size of 268435456 bytes exhausted (tried to allocate 20480 bytes) in /home/x/x.cz/htdocs/wp-content/plugins/elementor/includes/controls/base-multiple.php on line 59" while reading response header"#; + assert_eq!(parse_oom_limit_bytes(l), Some(256 * MIB)); + } + + #[test] + fn ignores_lines_that_only_look_similar() { + assert_eq!(parse_oom_limit_bytes("PHP Warning: something"), None); + assert_eq!( + parse_oom_limit_bytes("Allowed memory size of bytes exhausted"), + None + ); + assert_eq!(parse_oom_limit_bytes("Allowed memory size of 12abc"), None); + } + + #[test] + fn counts_only_fatals_at_the_current_limit_or_above() { + let text = "Allowed memory size of 268435456 bytes exhausted\n\ + Allowed memory size of 402653184 bytes exhausted\n\ + unrelated\n"; + assert_eq!(count_ooms(text, 256), 2); + // After a raise to 384, the 256 line is history. + assert_eq!(count_ooms(text, 384), 1); + assert_eq!(count_ooms(text, 512), 0); + } + + #[test] + fn raises_one_step_on_a_fresh_oom() { + let mut s = st(256); + s.last_oom_at = 1_000_000; + assert_eq!( + decide(1_000_000, 256, 512, None, &s), + Step::Raise { to: 384 } + ); + } + + #[test] + fn waits_out_the_cooldown_but_remembers_the_oom() { + let mut s = st(256); + s.last_change_at = 1_000_000; + s.last_oom_at = 1_000_100; + assert_eq!(decide(1_000_100, 384, 512, None, &s), Step::Nothing); + assert_eq!( + decide(1_000_000 + COOLDOWN_SECS, 384, 512, None, &s), + Step::Raise { to: 512 } + ); + } + + #[test] + fn never_raises_past_the_ceiling() { + let mut s = st(256); + s.last_oom_at = 10; + assert_eq!( + decide(COOLDOWN_SECS * 2, 448, 512, None, &s), + Step::Raise { to: 512 } + ); + assert_eq!( + decide(COOLDOWN_SECS * 2, 512, 512, None, &s), + Step::Capped { + at: 512, + by_ram: false + } + ); + } + + #[test] + fn node_ram_caps_below_the_ceiling() { + let mut s = st(256); + s.last_oom_at = 10; + // 2 GiB node, 5 workers ⇒ 307 MiB each. + let ram = ram_cap_mb(2048, 5); + assert_eq!(ram, Some(307)); + assert_eq!( + decide(COOLDOWN_SECS * 2, 256, 512, ram, &s), + Step::Raise { to: 307 } + ); + assert_eq!( + decide(COOLDOWN_SECS * 2, 307, 512, ram, &s), + Step::Capped { + at: 307, + by_ram: true + } + ); + } + + #[test] + fn capped_alert_is_throttled() { + let mut s = st(256); + s.last_oom_at = 100; + s.capped_alert_at = 50; + assert_eq!(decide(60, 512, 512, None, &s), Step::Nothing); + assert!(matches!( + decide(50 + CAPPED_ALERT_EVERY_SECS, 512, 512, None, &s), + Step::Capped { .. } + )); + } + + #[test] + fn lowers_one_step_after_a_quiet_fortnight_never_below_base() { + let mut s = st(256); + s.last_oom_at = 0; + s.last_change_at = 100; + assert_eq!( + decide(100 + LOWER_AFTER_SECS - 1, 512, 512, None, &s), + Step::Nothing + ); + assert_eq!( + decide(100 + LOWER_AFTER_SECS, 512, 512, None, &s), + Step::Lower { to: 384 } + ); + assert_eq!( + decide(100 + LOWER_AFTER_SECS, 300, 512, None, &s), + Step::Lower { to: 256 } + ); + assert_eq!( + decide(100 + LOWER_AFTER_SECS, 256, 512, None, &s), + Step::Nothing + ); + } + + #[test] + fn a_recent_oom_postpones_the_lowering() { + let mut s = st(256); + s.last_change_at = 100; + s.last_oom_at = 50; // answered (before the change) + assert!(!s.oom_pending()); + assert!(matches!( + decide(100 + LOWER_AFTER_SECS, 384, 512, None, &s), + Step::Lower { .. } + )); + } + + #[test] + fn a_lowered_ceiling_pulls_the_limit_down_but_not_under_base() { + let s = st(256); + assert_eq!(decide(0, 768, 512, None, &s), Step::Lower { to: 512 }); + // Ceiling under the operator's own value: their value wins. + let s = st(384); + assert_eq!(decide(0, 512, 256, None, &s), Step::Lower { to: 384 }); + assert_eq!(decide(0, 384, 256, None, &s), Step::Nothing); + } + + #[test] + fn state_round_trips_and_tolerates_missing_fields() { + let s = State { + base_mb: 256, + last_oom_at: 1, + last_change_at: 2, + capped_alert_at: 3, + log_ino: 4, + log_offset: 5, + }; + assert_eq!(State::parse(&s.to_json()), Some(s)); + assert_eq!(State::parse(r#"{"base_mb":128}"#), Some(st(128))); + assert_eq!(State::parse("garbage"), None); + } +}