feat(limits): raise PHP memory_limit automatically when a site runs out

bin/hyperion-agent/src/main.rs

@@ -1067,6 +1067,26 @@ async fn main() -> anyhow::Result<()> {
             }
         });
     }
+    // Automatic PHP memory_limit: answer "Allowed memory size … exhausted" in
+    // a site's error.log by raising its pool one step (bounded), and step it
+    // back down after a quiet fortnight. Its own two-minute task: a site
+    // erroring out should not wait for the five-minute loop, and reading
+    // tenant logs must never hold fail2ban or quota enforcement up.
+    {
+        let mem_svc = svc.clone();
+        tokio::spawn(async move {
+            tokio::time::sleep(std::time::Duration::from_secs(90)).await;
+            let mut interval = tokio::time::interval(std::time::Duration::from_secs(120));
+            loop {
+                interval.tick().await;
+                match mem_svc.php_mem_auto_tick().await {
+                    Ok(n) if n > 0 => tracing::info!(pools = n, "php mem auto: rewrote pools"),
+                    Ok(_) => {}
+                    Err(e) => tracing::warn!(error=%e, "php mem auto tick failed"),
+                }
+            }
+        });
+    }
     // The weekly page walk. On its OWN task, not in the five-minute loop:
     // one pass fetches up to eight pages plus forty links PER SITE, each
     // with a 20-second ceiling, so on a node with a few dozen care-plan

bin/hyperion-web/src/handlers/hostings.rs

@@ -126,6 +126,7 @@ struct DetailTpl<'a> {
     htmx_version: &'static str,
     detail: HostingDetail,
     limits: hyperion_types::HostingLimits,
+    mem_auto: PhpMemAutoView,
     wp_status: Option<WpInstallStatus>,
     expiry: hyperion_types::HostingExpiry,
     backups: Vec<hyperion_types::BackupRunWire>,
@@ -1394,6 +1395,7 @@ pub async fn post_create(
                 .unwrap_or_else(|_| hyperion_types::HostingLimits::defaults());
             let staging_domain_default = format!("staging.{}", detail.domain);
             let preview_domain = compute_preview_domain(&state, target, &detail.domain).await;
+            let mem_auto = PhpMemAutoView::from_kv(&[], limits.php_memory_mb);
             let tpl = DetailTpl {
                 username: &ctx.username,
                 user_initial: super::user_initial(&ctx.username),
@@ -1402,6 +1404,7 @@ pub async fn post_create(
                 htmx_version: super::htmx_version(),
                 detail,
                 limits,
+                mem_auto,
                 wp_status: None,
                 expiry: hyperion_types::HostingExpiry::defaults(),
                 backups: vec![],
@@ -1788,6 +1791,43 @@ pub(crate) fn compute_hosting_health(
     }
 }
 
+/// The automatic PHP memory_limit, as the OWNING node stores it
+/// (`hyperion_types::phpmem`).
+pub struct PhpMemAutoView {
+    pub enabled: bool,
+    /// The most the automation may set, in MiB.
+    pub max_mb: i64,
+    /// The operator's own limit, when the automation currently has the pool
+    /// above it.
+    pub raised_from_mb: Option<i64>,
+    /// Last out-of-memory fatal seen; 0 = none since it was turned on.
+    pub last_oom_at: i64,
+}
+
+impl PhpMemAutoView {
+    fn from_kv(pairs: &[(String, String)], current_mb: i64) -> Self {
+        use hyperion_types::phpmem;
+        let get = |k: &str| {
+            pairs
+                .iter()
+                .find(|(key, _)| key == k)
+                .map(|(_, v)| v.as_str())
+        };
+        let st = get(phpmem::KV_STATE).and_then(phpmem::State::parse);
+        PhpMemAutoView {
+            enabled: get(phpmem::KV_ENABLED) == Some("on"),
+            max_mb: get(phpmem::KV_MAX_MB)
+                .and_then(|v| v.trim().parse().ok())
+                .unwrap_or(phpmem::DEFAULT_MAX_MB),
+            raised_from_mb: st
+                .as_ref()
+                .map(|s| s.base_mb)
+                .filter(|&b| b > 0 && b < current_mb),
+            last_oom_at: st.map(|s| s.last_oom_at).unwrap_or(0),
+        }
+    }
+}
+
 /// Operator-tunable php.ini settings, applied via a per-hosting
 /// `.user.ini` in htdocs (PHP_INI_PERDIR — no FPM pool change, so a bad
 /// value can never stop the pool from starting). Empty string = leave
@@ -2213,6 +2253,7 @@ pub async fn get_detail(
         Ok(RpcResponse::HostingKvList(v)) => v,
         _ => vec![],
     };
+    let mem_auto = PhpMemAutoView::from_kv(&owner_kv, limits.php_memory_mb);
     let backup_cadence = owner_kv
         .iter()
         .find(|(k, _)| k == "backup_cadence")
@@ -2398,6 +2439,7 @@ pub async fn get_detail(
         htmx_version: super::htmx_version(),
         detail,
         limits,
+        mem_auto,
         wp_status,
         expiry,
         backups,
@@ -4521,6 +4563,11 @@ pub struct SetLimitsForm {
     php_max_children: i64,
     php_max_requests: i64,
     db_max_connections: i64,
+    /// Checkbox: absent when unticked.
+    #[serde(default)]
+    mem_auto: String,
+    #[serde(default)]
+    mem_auto_max_mb: String,
     #[serde(default)]
     target_node: String,
 }
@@ -5007,13 +5054,68 @@ pub async fn post_set_limits(
     // The enforced caps are the hosting_quotas row (the quota card), which
     // profiles also write through quota_set. Saving this card now clears
     // any stale ghost value, so old rows converge on the truth.
+    let mem_auto_max: i64 = match form.mem_auto_max_mb.trim() {
+        "" => hyperion_types::phpmem::DEFAULT_MAX_MB,
+        v => match v.parse::<i64>() {
+            Ok(n) if (16..=8192).contains(&n) => n,
+            _ => {
+                return Ok(save_result(
+                    &headers,
+                    false,
+                    "Automatic memory ceiling must be 16–8192 MB.",
+                    format!(
+                        "/hostings/{}?flash_error={}#limits",
+                        urlencoding(&form.selector),
+                        urlencoding("Automatic memory ceiling must be 16–8192 MB.")
+                    ),
+                ))
+            }
+        },
+    };
+    let mem_auto_on = form.mem_auto.trim() == "on";
     let target = node_target(&form.target_node);
     let resp = crate::dispatcher::dispatch_to_node(
         &state,
         target,
-        Request::HostingSetLimits { sel, limits: l },
+        Request::HostingSetLimits {
+            sel: sel.clone(),
+            limits: l,
+        },
     )
     .await?;
+    if let RpcResponse::HostingSetLimits(_) = resp {
+        // The automation's switch and ceiling live in the OWNING node's
+        // hosting_kv, beside the state its tick keeps — the pool it rewrites
+        // is there. Written after the limits, so a refused save above leaves
+        // them as they were.
+        let hosting_id = match find_hosting_anywhere(&state, sel).await {
+            Ok((d, _)) => d.id.as_str().to_string(),
+            Err(e) => return Err(e),
+        };
+        for (key, value) in [
+            (
+                hyperion_types::phpmem::KV_ENABLED,
+                if mem_auto_on { "on" } else { "off" }.to_string(),
+            ),
+            (hyperion_types::phpmem::KV_MAX_MB, mem_auto_max.to_string()),
+        ] {
+            match crate::dispatcher::dispatch_to_node(
+                &state,
+                target,
+                Request::HostingKvSet {
+                    hosting_id: hosting_id.clone(),
+                    key: key.into(),
+                    value,
+                },
+            )
+            .await?
+            {
+                RpcResponse::HostingKvSet => {}
+                RpcResponse::Error(e) => return Err(AppError::Rpc(e.to_string())),
+                _ => return Err(AppError::Internal("unexpected response".into())),
+            }
+        }
+    }
     match resp {
         RpcResponse::HostingSetLimits(_) => Ok(save_result(
             &headers,

bin/hyperion-web/templates/hostings_detail.html

@@ -2763,6 +2763,30 @@ <h2>
                value="{{ limits.db_max_connections }}">
       </div>
     </div>
+    {# Automatic memory_limit: the owning node watches this site's error.log
+       for "Allowed memory size … exhausted" and raises the pool one 128 MB
+       step at a time up to the ceiling (and what the node's RAM allows);
+       two quiet weeks take it one step back. Unticking restores the value
+       above. State lives in the owning node's hosting_kv. #}
+    <div class="field-row">
+      <div class="field">
+        <label class="field-checkbox" style="margin-top:1.6rem">
+          <input type="checkbox" name="mem_auto" value="on"{% if mem_auto.enabled %} checked{% endif %}>
+          <span>Raise PHP memory automatically when the site runs out <span class="hint">— +128 MB per step, back down after 2 quiet weeks</span></span>
+        </label>
+      </div>
+      <div class="field">
+        <label for="mem_auto_max_mb">Automatic ceiling <span class="hint">MB</span></label>
+        <input id="mem_auto_max_mb" name="mem_auto_max_mb" type="number" min="16" max="8192"
+               value="{{ mem_auto.max_mb }}">
+      </div>
+    </div>
+    {% if mem_auto.enabled %}
+    <p class="note" style="margin:0 0 0.8rem">
+      {% if let Some(from) = mem_auto.raised_from_mb %}Raised automatically from your {{ from }} MB to {{ limits.php_memory_mb }} MB.{% else %}At your own value.{% endif %}
+      {% if mem_auto.last_oom_at > 0 %}Last ran out of memory {{ crate::handlers::stats::fmt_ago(mem_auto.last_oom_at) }}.{% else %}No out-of-memory errors seen since it was turned on.{% endif %}
+    </p>
+    {% endif %}
     {# Disk and bandwidth caps are deliberately NOT here. This card once
        carried its own "Disk limit" and "Monthly bandwidth" fields, and they
        were placebo: stored in a column nothing enforced and nothing alerted

crates/hyperion-adapters/src/fs.rs

@@ -81,9 +81,12 @@ pub fn open_no_follow(path: &Path) -> Result<std::fs::File, AdapterError> {
     use rustix::fs::{Mode, OFlags};
     use std::os::unix::fs::MetadataExt;
 
+    // NONBLOCK: a FIFO in place of the log would otherwise block the open
+    // itself — before the regular-file check below could refuse it — and
+    // hang whichever loop asked. It changes nothing for a regular file.
     let fd = rustix::fs::open(
         path,
-        OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
+        OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC | OFlags::NONBLOCK,
         Mode::empty(),
     )
     .map_err(|e| {
@@ -196,6 +199,75 @@ pub fn truncate_no_follow(path: &Path) -> Result<(), AdapterError> {
 ///
 /// Replaces shelling out to `tail`, which takes a PATH and therefore resolves
 /// it again — after any check the caller made, and as root.
+/// What [`read_appended`] found: the new text plus where to resume.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Appended {
+    pub text: String,
+    pub ino: u64,
+    pub offset: u64,
+}
+
+/// Read what was appended to a log since `(ino, offset)`, the way `tail -F`
+/// would, opened through [`open_no_follow`] (the logs live in a directory
+/// the site user owns).
+///
+/// * First sight (`ino == 0`): nothing is returned and the position is set
+///   to the end — history is not news.
+/// * Rotated (inode changed) or truncated (shorter than `offset`): read the
+///   new file from its start.
+/// * More than `max` bytes new: only the newest `max` are read.
+///
+/// The returned offset stops after the last complete line, so a line nginx
+/// is still writing is read whole next time instead of half now.
+pub fn read_appended(
+    path: &Path,
+    ino: u64,
+    offset: u64,
+    max: u64,
+) -> Result<Appended, AdapterError> {
+    use std::io::{Read, Seek, SeekFrom};
+    use std::os::unix::fs::MetadataExt;
+
+    let mut file = open_no_follow(path)?;
+    let md = file
+        .metadata()
+        .map_err(|e| AdapterError::Other(format!("stat: {e}")))?;
+    let (cur_ino, len) = (md.ino(), md.len());
+    if ino == 0 {
+        return Ok(Appended {
+            text: String::new(),
+            ino: cur_ino,
+            offset: len,
+        });
+    }
+    let from = if cur_ino != ino || len < offset {
+        0
+    } else {
+        offset
+    };
+    let start = from.max(len.saturating_sub(max));
+    if start >= len {
+        return Ok(Appended {
+            text: String::new(),
+            ino: cur_ino,
+            offset: len,
+        });
+    }
+    file.seek(SeekFrom::Start(start))
+        .map_err(|e| AdapterError::Other(format!("seek: {e}")))?;
+    let mut buf = Vec::with_capacity((len - start) as usize);
+    file.take(len - start)
+        .read_to_end(&mut buf)
+        .map_err(|e| AdapterError::Other(format!("read: {e}")))?;
+    let complete = buf.iter().rposition(|&b| b == b'\n').map_or(0, |i| i + 1);
+    buf.truncate(complete);
+    Ok(Appended {
+        text: String::from_utf8_lossy(&buf).into_owned(),
+        ino: cur_ino,
+        offset: start + complete as u64,
+    })
+}
+
 pub async fn tail_lines(path: &Path, lines: usize) -> Result<String, AdapterError> {
     use std::io::{Read, Seek, SeekFrom};
 
@@ -449,6 +521,58 @@ fn with_extension(p: &Path, ext: &str) -> PathBuf {
 mod tests {
     use super::*;
 
+    #[test]
+    fn read_appended_follows_a_growing_log() {
+        use std::io::Write;
+        let d = tempfile::tempdir().expect("tempdir");
+        let p = d.path().join("error.log");
+        std::fs::write(&p, "old line\n").expect("write");
+
+        // First sight: history is skipped.
+        let a = read_appended(&p, 0, 0, 1 << 20).expect("read");
+        assert_eq!(a.text, "");
+        assert_eq!(a.offset, 9);
+
+        // Appended text, with a half-written last line held back.
+        let mut f = std::fs::OpenOptions::new()
+            .append(true)
+            .open(&p)
+            .expect("open");
+        f.write_all(b"one\ntwo\npart").expect("append");
+        let b = read_appended(&p, a.ino, a.offset, 1 << 20).expect("read");
+        assert_eq!(b.text, "one\ntwo\n");
+        assert_eq!(b.offset, 17);
+
+        // The finished line arrives next time.
+        f.write_all(b"ial\n").expect("append");
+        let c = read_appended(&p, b.ino, b.offset, 1 << 20).expect("read");
+        assert_eq!(c.text, "partial\n");
+
+        // Truncated (rotation by copytruncate): read from the start.
+        std::fs::write(&p, "fresh\n").expect("write");
+        let e = read_appended(&p, c.ino, c.offset, 1 << 20).expect("read");
+        assert_eq!(e.text, "fresh\n");
+
+        // A burst larger than `max`: only the newest part.
+        let mut f = std::fs::OpenOptions::new()
+            .append(true)
+            .open(&p)
+            .expect("open");
+        f.write_all(b"aaaaaaaaaa\nbbbb\n").expect("append");
+        let g = read_appended(&p, e.ino, e.offset, 8).expect("read");
+        assert_eq!(g.text, "aa\nbbbb\n");
+    }
+
+    #[test]
+    fn read_appended_refuses_a_symlink() {
+        let d = tempfile::tempdir().expect("tempdir");
+        let target = d.path().join("secret");
+        std::fs::write(&target, "x\n").expect("write");
+        let link = d.path().join("error.log");
+        std::os::unix::fs::symlink(&target, &link).expect("symlink");
+        assert!(read_appended(&link, 1, 0, 1 << 20).is_err());
+    }
+
     #[tokio::test]
     async fn atomic_write_creates_parent_and_file() {
         let d = tempfile::tempdir().expect("tempdir");