diff --git a/crates/app/src/cmd_status.rs b/crates/app/src/cmd_status.rs index e48cd9cad..1389f09fb 100755 --- a/crates/app/src/cmd_status.rs +++ b/crates/app/src/cmd_status.rs @@ -46,17 +46,8 @@ pub fn run(args: &StatusArgs) { |_| extenddb_config::pid_file_path_default(port), |c| extenddb_config::pid_file_path(&config::expand_tilde(&c.server.run_dir), port), ); - let pid_label = std::fs::read_to_string(&pid_file) - .ok() - .and_then(|contents| { - let pid = contents.trim().to_owned(); - let pid_num: i32 = pid.parse().ok()?; - if crate::util::is_process_alive(pid_num) { - Some(pid) - } else { - None - } - }); + let pid_label = server_pid(&pid_file) + .filter(|pid| pid.parse::().is_ok_and(crate::util::is_process_alive)); match pid_label { Some(pid) => println!("extenddb is running on port {port} (pid {pid})"), None => println!("extenddb is running on port {port} (pid unknown)"), @@ -71,6 +62,22 @@ pub fn run(args: &StatusArgs) { } } +/// The PID in the PID file, when a running server holds the file. A file +/// nothing holds names a PID the system may have given to another process. +#[cfg(unix)] +fn server_pid(path: &std::path::Path) -> Option { + let contents = extenddb_server::pid_file::read(path).ok()?; + contents.held.then(|| contents.text.trim().to_owned()) +} + +/// Non-unix: PID files only exist in daemon mode, which is unix-only. +#[cfg(not(unix))] +fn server_pid(path: &std::path::Path) -> Option { + std::fs::read_to_string(path) + .ok() + .map(|s| s.trim().to_owned()) +} + /// Determine the port to check: CLI override > config file > default. fn resolve_port(args: &StatusArgs) -> anyhow::Result { if let Some(port) = args.port { diff --git a/crates/app/src/cmd_stop.rs b/crates/app/src/cmd_stop.rs index a3963ac0c..4b214d8f1 100755 --- a/crates/app/src/cmd_stop.rs +++ b/crates/app/src/cmd_stop.rs @@ -3,8 +3,9 @@ //! `extenddb stop` — stop the running extenddb server. //! -//! Reads the PID file, sends SIGTERM, polls until the process exits (or -//! times out), and cleans up the stale PID file. +//! Reads the PID file, checks that a running server holds it, sends SIGTERM, +//! polls until the process exits (or times out), and cleans up the stale PID +//! file. #[cfg(unix)] use std::thread; @@ -76,8 +77,11 @@ pub fn run(args: &StopArgs) { None => extenddb_config::pid_file_path_default(port), }; - let pid_str = match std::fs::read_to_string(&pid_file) { - Ok(s) => s.trim().to_owned(), + // The PID and whether a running server holds the file, from one open of + // it, so a server that replaces the file meanwhile cannot pair its lock + // with the stale PID. + let contents = match extenddb_server::pid_file::read(&pid_file) { + Ok(contents) => contents, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { // `serve --foreground` writes no PID file, so a missing file does // not mean nothing is running. Probe the port before saying so. @@ -104,6 +108,7 @@ pub fn run(args: &StopArgs) { std::process::exit(1); } }; + let pid_str = contents.text.trim(); let pid: i32 = if let Ok(p) = pid_str.parse() { p @@ -112,12 +117,51 @@ pub fn run(args: &StopArgs) { std::process::exit(1); }; - // Check if the process is alive before sending SIGTERM. - if !is_process_alive(pid) { - eprintln!("Process {pid} is not running. Cleaning up stale PID file."); - let _ = std::fs::remove_file(&pid_file); - // Exit 0: the user's intent ("stop extenddb") is satisfied — the server - // is not running. This makes scripting easier (e.g. `extenddb stop && ...`). + // A running server holds a lock on its PID file. When nothing holds it, the + // PID is stale: the server is gone and the system may have given the PID to + // another process, which must not be signalled. The lock is checked first + // because `is_process_alive` reports a server another user runs as gone. + if !contents.held { + if !is_process_alive(pid) { + eprintln!("Process {pid} is not running. Cleaning up stale PID file."); + let _ = extenddb_server::pid_file::remove_stale(&pid_file, &contents); + // Exit 0: the user's intent ("stop extenddb") is satisfied, since + // the server is not running. This makes scripting easier (e.g. + // `extenddb stop && ...`). + return; + } + // The address `serve` would bind with the same config file and + // `EXTENDDB__` variables, including when there is no file. + let bind_addr = app_config + .as_ref() + .map(|c| c.server.bind_addr.clone()) + .or_else(|| { + extenddb_config::load_builtin_defaults() + .ok() + .map(|c| c.server.bind_addr) + }) + .unwrap_or_else(|| extenddb_config::ServerConfig::default().bind_addr); + let probed = listen_probe_addrs(&bind_addr, port); + if let Some(addr) = probed.iter().find(|a| is_listening(a)) { + eprintln!( + "PID file {} names process {pid}, but no extenddb server holds the file, so \ + `stop` cannot tell whether that process is the server listening on {addr}, \ + and does not signal it.\n\ + A server that an earlier extenddb release started with `serve --foreground \ + --write-pid-file` holds no lock on its PID file. If process {pid} is that \ + server, stop it with: kill {pid}", + pid_file.display(), + ); + std::process::exit(1); + } + let probed: Vec = probed.iter().map(ToString::to_string).collect(); + eprintln!( + "PID file {} names process {pid}, which is not an extenddb server: no server \ + holds the file and nothing listens on {}. Cleaning up stale PID file.", + pid_file.display(), + probed.join(", "), + ); + let _ = extenddb_server::pid_file::remove_stale(&pid_file, &contents); return; } @@ -135,8 +179,9 @@ pub fn run(args: &StopArgs) { let deadline = Instant::now() + SHUTDOWN_TIMEOUT; loop { if !is_process_alive(pid) { - // Process exited — clean up PID file if the server didn't already. - let _ = std::fs::remove_file(&pid_file); + // Process exited: clean up its PID file if the server didn't + // already, but not a file a new server has put in its place. + let _ = extenddb_server::pid_file::remove_stale(&pid_file, &contents); println!("extenddb stopped (pid {pid})"); return; } @@ -158,11 +203,83 @@ pub fn run(args: &StopArgs) { } } +/// The addresses a server on `port` configured with `bind_addr` answers on: +/// that address, both loopback addresses for a wildcard one, the addresses a +/// host name resolves to, and 127.0.0.1 in every case. +#[cfg(unix)] +fn listen_probe_addrs(bind_addr: &str, port: u16) -> Vec { + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, ToSocketAddrs}; + + // serve binds `{bind_addr}:{port}`, so it also accepts an IPv6 address in + // brackets. + let host = bind_addr + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(bind_addr); + let mut addrs = vec![SocketAddr::from((Ipv4Addr::LOCALHOST, port))]; + match host.parse::() { + Ok(ip) if ip.is_unspecified() => addrs.push(SocketAddr::from((Ipv6Addr::LOCALHOST, port))), + Ok(ip) => addrs.push(SocketAddr::new(ip, port)), + Err(_) => addrs.extend((host, port).to_socket_addrs().into_iter().flatten()), + } + addrs.sort_unstable(); + addrs.dedup(); + addrs +} + +/// Whether something accepts a TCP connection on `addr`. +#[cfg(unix)] +fn is_listening(addr: &std::net::SocketAddr) -> bool { + std::net::TcpStream::connect_timeout(addr, Duration::from_millis(500)).is_ok() +} + #[cfg(all(test, unix))] mod tests { - use super::SHUTDOWN_TIMEOUT; + use std::net::SocketAddr; use std::time::Duration; + use super::{SHUTDOWN_TIMEOUT, listen_probe_addrs}; + + fn addrs(list: &[&str]) -> Vec { + let mut v: Vec = list.iter().map(|a| a.parse().unwrap()).collect(); + v.sort_unstable(); + v + } + + #[test] + fn a_specific_address_is_probed_with_loopback() { + assert_eq!( + listen_probe_addrs("10.1.2.3", 18443), + addrs(&["10.1.2.3:18443", "127.0.0.1:18443"]) + ); + for ipv6 in ["::1", "[::1]"] { + assert_eq!( + listen_probe_addrs(ipv6, 18443), + addrs(&["[::1]:18443", "127.0.0.1:18443"]), + "{ipv6}" + ); + } + } + + #[test] + fn a_wildcard_address_is_probed_on_both_loopbacks() { + for wildcard in ["0.0.0.0", "::", "[::]"] { + assert_eq!( + listen_probe_addrs(wildcard, 18443), + addrs(&["127.0.0.1:18443", "[::1]:18443"]), + "{wildcard}" + ); + } + } + + #[test] + fn the_default_bind_address_is_probed_once() { + assert_eq!( + listen_probe_addrs("127.0.0.1", 18443), + addrs(&["127.0.0.1:18443"]) + ); + } + /// `stop` waits at least as long as the server's shutdown can take with /// a backup job in its failure path: 5 s of connection drain, 30 s of /// job drain, 36 s after it (10 s for the supervisor's return, 16 s for diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index f723b3790..14460cbb3 100644 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -24,6 +24,8 @@ mod leading_keys; mod listener; pub mod management; mod metrics_endpoint; +#[cfg(unix)] +pub mod pid_file; pub mod rate_limit; mod registration; mod repeated_headers; diff --git a/crates/server/src/pid_file.rs b/crates/server/src/pid_file.rs new file mode 100644 index 000000000..eb1083b58 --- /dev/null +++ b/crates/server/src/pid_file.rs @@ -0,0 +1,355 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The PID file that `extenddb status` and `extenddb stop` read. +//! +//! A running server holds an exclusive `flock` on its PID file: `daemonize` +//! takes it in daemon mode, and [`hold`] takes it in foreground mode. A reader +//! that finds the lock free knows that no server owns the file, so the PID in +//! it is stale even when a process with that PID is alive, for example after +//! the server crashed and the system gave its PID to another process. +//! +//! `flock` locks belong to open file descriptions, not to processes, so two +//! descriptors of the same file conflict even inside one process. [`read`] +//! takes the PID and the lock state from one descriptor, and [`hold`] writes a +//! new file and renames it into place, so a reader sees either an old file no +//! process holds or the server's file, complete and held. In a run directory +//! the server cannot write, [`hold`] rewrites an existing file in place. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::MetadataExt; +use std::path::Path; + +/// What [`hold`] did with the PID file. +#[derive(Debug)] +pub enum Hold { + /// This process wrote the file and holds its lock for as long as the + /// `File` stays open. + Taken(File), + /// The file already names this process and is locked: in daemon mode + /// `daemonize` wrote it and keeps the lock on its own descriptor. + Daemonized, + /// Another running process holds the file, for example a server on the + /// same port bound to another address. The file is left as it is; the + /// value is the PID it names. + HeldByOther(String), +} + +/// A PID file as one open of it saw it. +#[derive(Debug)] +pub struct Contents { + /// The file's text, normally a PID and a newline. + pub text: String, + /// Whether a process held the file's lock, that is, whether a running + /// server owns it. + pub held: bool, + dev: u64, + ino: u64, +} + +/// Write this process's PID to `path` and hold an exclusive lock on the file. +/// +/// The PID goes to a new file that is renamed into place. When the run +/// directory cannot be written but the PID file can, as when an init system +/// created the file for the server, the file is rewritten in place, as +/// earlier releases did. +/// +/// # Errors +/// +/// Returns the I/O error when the existing file cannot be read or the new +/// file cannot be created, locked, written, or renamed into place. +pub fn hold(path: &Path) -> io::Result { + let pid = std::process::id().to_string(); + match read(path) { + Ok(existing) if existing.held => { + let named = existing.text.trim(); + return Ok(if named == pid { + Hold::Daemonized + } else { + Hold::HeldByOther(named.to_owned()) + }); + } + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + let tmp = path.with_extension(format!("pid.{pid}.tmp")); + let write = || -> io::Result { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&tmp)?; + try_lock(&file, libc::LOCK_EX)?; + file.write_all(format!("{pid}\n").as_bytes())?; + std::fs::rename(&tmp, path)?; + Ok(file) + }; + match write() { + Ok(file) => Ok(Hold::Taken(file)), + Err(e) => { + let _ = std::fs::remove_file(&tmp); + if e.kind() == io::ErrorKind::PermissionDenied && path.exists() { + return rewrite(path, &pid).map(Hold::Taken); + } + Err(e) + } + } +} + +/// Lock the existing file at `path` and replace its text with `pid`. A reader +/// that read the old text can then remove the file this server holds, which +/// the rename avoids, but only a process that can write the directory can +/// remove a file in it. +fn rewrite(path: &Path, pid: &str) -> io::Result { + let mut file = OpenOptions::new().write(true).open(path)?; + try_lock(&file, libc::LOCK_EX)?; + file.set_len(0)?; + file.write_all(format!("{pid}\n").as_bytes())?; + Ok(file) +} + +/// Read the PID file at `path` and probe its lock through the same open. +/// +/// # Errors +/// +/// Returns the I/O error when the file cannot be opened, read, or probed; a +/// missing file is `NotFound`. +pub fn read(path: &Path) -> io::Result { + let mut file = File::open(path)?; + let held = match try_lock(&file, libc::LOCK_SH) { + Ok(()) => { + // Free, so no server owns it. Release the shared lock at once. A + // foreground server locks its new file before renaming it into + // place, so it never meets this lock. `daemonize` opens the file + // at its path and then locks it without waiting, so a daemon that + // starts in the two calls this lock lasts fails to start. + unlock(&file)?; + false + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => true, + Err(e) => return Err(e), + }; + let mut text = String::new(); + file.read_to_string(&mut text)?; + let meta = file.metadata()?; + Ok(Contents { + text, + held, + dev: meta.dev(), + ino: meta.ino(), + }) +} + +/// Remove the stale PID file at `path`, unless the path now names another +/// file than the one `contents` was read from, as it does once a new server +/// has renamed its own file into place. +/// +/// # Errors +/// +/// Returns the I/O error of the removal; a file already gone is not an error. +pub fn remove_stale(path: &Path, contents: &Contents) -> io::Result<()> { + match std::fs::metadata(path) { + Ok(meta) if meta.dev() == contents.dev && meta.ino() == contents.ino => { + match std::fs::remove_file(path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } + } + Ok(_) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +fn try_lock(file: &File, operation: libc::c_int) -> io::Result<()> { + // SAFETY: `flock` reads only the descriptor, which `file` keeps open for + // the duration of the call. + if unsafe { libc::flock(file.as_raw_fd(), operation | libc::LOCK_NB) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +fn unlock(file: &File) -> io::Result<()> { + // SAFETY: as in `try_lock`. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_UN) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::os::fd::AsRawFd; + use std::os::unix::fs::MetadataExt; + use std::path::PathBuf; + + use super::{Hold, hold, read, remove_stale}; + + fn scratch(name: &str) -> (PathBuf, PathBuf) { + let dir = std::env::temp_dir().join(format!( + "extenddb-pid-file-{name}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + std::fs::create_dir(&dir).expect("scratch dir"); + let path = dir.join("extenddb-18443.pid"); + (dir, path) + } + + fn own_pid_line() -> String { + format!("{}\n", std::process::id()) + } + + /// A file locked on its own descriptor, as `daemonize` or another server + /// leaves it. + fn locked_file(path: &PathBuf, text: &str) -> File { + let file = File::create(path).unwrap(); + std::fs::write(path, text).unwrap(); + // SAFETY: `file` keeps the descriptor open for the call. + assert_eq!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }, + 0 + ); + file + } + + fn taken(result: Hold) -> File { + match result { + Hold::Taken(file) => file, + other => panic!("expected Taken, got {other:?}"), + } + } + + /// The file names this process while it is held, and a reader sees the + /// lock until the holder is dropped. + #[test] + fn a_held_pid_file_is_seen_as_held_until_it_is_dropped() { + let (dir, path) = scratch("held"); + let held = taken(hold(&path).expect("hold")); + let seen = read(&path).unwrap(); + assert_eq!(seen.text, own_pid_line()); + assert!(seen.held); + drop(held); + assert!(!read(&path).unwrap().held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A PID file left by a server that is gone holds no lock, whatever PID it + /// names, and a new server replaces it with a new file. + #[test] + fn a_stale_pid_file_is_free_and_is_replaced() { + let (dir, path) = scratch("stale"); + std::fs::write(&path, "1\n").unwrap(); + let old_ino = std::fs::metadata(&path).unwrap().ino(); + assert!(!read(&path).unwrap().held); + let held = taken(hold(&path).expect("hold")); + assert_eq!(std::fs::read_to_string(&path).unwrap(), own_pid_line()); + assert_ne!(std::fs::metadata(&path).unwrap().ino(), old_ino); + assert!(read(&path).unwrap().held); + drop(held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A PID file the server can write, in a run directory it cannot, is + /// rewritten in place and held. + #[test] + fn a_writable_file_in_a_read_only_directory_is_rewritten_in_place() { + use std::os::unix::fs::PermissionsExt; + + // Root writes any directory, so the rename path would run instead. + // SAFETY: `geteuid` takes no arguments and cannot fail. + if unsafe { libc::geteuid() } == 0 { + return; + } + let (dir, path) = scratch("read-only-dir"); + std::fs::write(&path, "1\n").unwrap(); + let ino = std::fs::metadata(&path).unwrap().ino(); + let mode = |m| std::fs::Permissions::from_mode(m); + std::fs::set_permissions(&dir, mode(0o555)).unwrap(); + let result = hold(&path); + std::fs::set_permissions(&dir, mode(0o755)).unwrap(); + let held = taken(result.expect("hold")); + assert_eq!(std::fs::read_to_string(&path).unwrap(), own_pid_line()); + assert_eq!(std::fs::metadata(&path).unwrap().ino(), ino); + assert!(read(&path).unwrap().held); + drop(held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A free file that names this process by coincidence, after the system + /// reused the PID of an earlier server, is not mistaken for daemonize's. + #[test] + fn a_free_file_naming_this_process_is_taken_over() { + let (dir, path) = scratch("own-stale"); + std::fs::write(&path, own_pid_line()).unwrap(); + let held = taken(hold(&path).expect("hold")); + assert!(read(&path).unwrap().held); + drop(held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// In daemon mode `daemonize` has already written this PID and keeps the + /// lock on its own descriptor; `hold` leaves both alone. + #[test] + fn a_file_daemonize_holds_for_this_process_is_left_to_it() { + let (dir, path) = scratch("daemon"); + let daemon = locked_file(&path, &own_pid_line()); + assert!(matches!(hold(&path).expect("hold"), Hold::Daemonized)); + assert!(read(&path).unwrap().held); + drop(daemon); + assert!(!read(&path).unwrap().held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A file another running process holds is not touched. + #[test] + fn a_file_another_process_holds_is_left_alone() { + let (dir, path) = scratch("other"); + let other = locked_file(&path, "424242\n"); + match hold(&path).expect("hold") { + Hold::HeldByOther(pid) => assert_eq!(pid, "424242"), + other => panic!("expected HeldByOther, got {other:?}"), + } + assert_eq!(std::fs::read_to_string(&path).unwrap(), "424242\n"); + drop(other); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A stale file is removed, but not a new file that a server renamed into + /// its place after the stale one was read. + #[test] + fn remove_stale_removes_only_the_file_that_was_read() { + let (dir, path) = scratch("remove"); + std::fs::write(&path, "1\n").unwrap(); + let stale = read(&path).unwrap(); + let held = taken(hold(&path).expect("hold")); + remove_stale(&path, &stale).unwrap(); + assert!(path.exists(), "the new server's file must stay"); + drop(held); + let stale = read(&path).unwrap(); + remove_stale(&path, &stale).unwrap(); + assert!(!path.exists()); + remove_stale(&path, &stale).unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn a_missing_pid_file_is_not_found() { + let (dir, path) = scratch("missing"); + assert_eq!( + read(&path).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index f1554b677..aac9b85f5 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -292,7 +292,12 @@ pub async fn serve(params: ServeParams) -> anyhow::Result<()> { let log_target = params.log_target; let result = serve_inner(params, port).await; if let Err(ref e) = result { - if let Some(ref path) = pid_path { + // Only this process's own file: another running server may hold the + // path when the two share a port number on different addresses. + if let Some(ref path) = pid_path + && std::fs::read_to_string(path) + .is_ok_and(|s| s.trim() == std::process::id().to_string()) + { let _ = std::fs::remove_file(path); } // P57 Bug 7: Log fatal errors where the operator will see them. After @@ -308,6 +313,63 @@ pub async fn serve(params: ServeParams) -> anyhow::Result<()> { result } +/// The PID file a server writes and the lock it keeps on it. +struct PidFileGuard { + /// The file this server owns, which `start_server` removes at shutdown. + path: Option, + /// Keeps the lock on the file this process wrote; `None` when there is no + /// file, or when `daemonize` holds the lock. + lock: Option, + /// The path and PID of a file another running process holds instead. + held_by_other: Option, +} + +#[cfg(unix)] +fn take_pid_file(pid_file: Option) -> anyhow::Result { + use crate::pid_file::Hold; + + let Some(path) = pid_file else { + return Ok(PidFileGuard { + path: None, + lock: None, + held_by_other: None, + }); + }; + let hold = crate::pid_file::hold(&path) + .map_err(|e| anyhow::anyhow!("Failed to write PID file {}: {e}", path.display()))?; + Ok(match hold { + Hold::Taken(file) => PidFileGuard { + path: Some(path), + lock: Some(file), + held_by_other: None, + }, + Hold::Daemonized => PidFileGuard { + path: Some(path), + lock: None, + held_by_other: None, + }, + Hold::HeldByOther(pid) => PidFileGuard { + held_by_other: Some(format!("{} (pid {pid})", path.display())), + path: None, + lock: None, + }, + }) +} + +/// Non-unix: no locks, and `stop` and `status` do not read PID files. +#[cfg(not(unix))] +fn take_pid_file(pid_file: Option) -> anyhow::Result { + if let Some(ref path) = pid_file { + std::fs::write(path, format!("{}\n", std::process::id())) + .map_err(|e| anyhow::anyhow!("Failed to write PID file {}: {e}", path.display()))?; + } + Ok(PidFileGuard { + path: pid_file, + lock: None, + held_by_other: None, + }) +} + /// Inner serve function — separated so [`serve`] can clean up the PID file on /// any error path. async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { @@ -323,15 +385,16 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let catalog_version = extenddb_storage::migrations::binary_catalog_label(); // Write the PID file, when the caller asked for one, so `extenddb status`, - // `extenddb stop`, and `start_server`'s graceful shutdown cleanup work. When - // the caller daemonized, the grandchild PID that `daemonize` wrote is the - // same value as `std::process::id()` post-fork, so this is a consistent - // rewrite rather than a conflicting one. A supervised deployment passes - // `None` and needs no writable run directory at all. - if let Some(ref path) = pid_file { - std::fs::write(path, format!("{}\n", std::process::id())) - .map_err(|e| anyhow::anyhow!("Failed to write PID file {}: {e}", path.display()))?; - } + // `extenddb stop`, and `start_server`'s graceful shutdown cleanup work. A + // supervised deployment passes `None` and needs no writable run directory + // at all. On unix the server holds a lock on the file until it exits, which + // is how `status` and `stop` tell its PID from a stale one; in daemon mode + // `daemonize` has already written the same PID and holds that lock. + let PidFileGuard { + path: pid_file, + lock: _pid_file_lock, + held_by_other, + } = take_pid_file(pid_file)?; // Init logging (REQ-LOG-003, REQ-LOG-006) — the caller chose the target via // [`LogTarget`]; a supervised/container deployment picks stderr so the @@ -400,6 +463,13 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { .try_init() .map_err(|e| anyhow::anyhow!("Failed to initialize tracing: {e}"))?; + if let Some(other) = held_by_other { + tracing::warn!( + "PID file {other} belongs to another running server; serving without a PID file, \ + so `extenddb status` and `extenddb stop` find that server, not this one" + ); + } + for warning in app_config.storage.startup_warnings() { tracing::warn!("{}", warning.log_message()); } diff --git a/devtools/run-tests b/devtools/run-tests index e7e2aa540..21789ee3b 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -440,7 +440,9 @@ if $NEEDS_INTEGRATION && [[ "$TARGET" != "real-dynamodb" ]]; then if ! "$BINARY" stop --config "$CONFIG_FOR_SETTINGS"; then echo " ✗ Could not stop the running server to apply the config change." echo " If it was started with 'serve --foreground', restart it with" - echo " --write-pid-file so 'stop' can signal it." + echo " --write-pid-file so 'stop' can signal it. A server that an earlier" + echo " release started with --write-pid-file holds no lock on its PID file;" + echo " stop it with the kill command 'stop' printed above." exit 1 fi sleep 1 diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index ec69d055c..9def5fa01 100755 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -254,6 +254,18 @@ journalctl -t extenddb -f **Fix:** Delete the PID file manually and use `extenddb status` or `ps` to find the extenddb process. Stop it with `kill `. +### `PID file names process , which is not an extenddb server` + +**Cause:** A running server holds a lock on its PID file, and nothing holds this one. The process it names is alive, but nothing accepts connections on the port at the bind address `stop` reads from the config file and the `EXTENDDB__` variables, or at 127.0.0.1. `stop` takes the process for one the system gave the PID to after the server exited without removing its file, for example after a crash. It does not signal it, removes the stale file, and exits 0. + +**Fix:** If no server should be running, none. A server that an earlier release started with `serve --foreground --write-pid-file`, bound to an address the config `stop` read does not name, also holds no lock: if `extenddb serve` then fails to bind the port, find that server with `ss -ltnp` and stop it with `kill `. + +### `PID file names process , but no extenddb server holds the file` + +**Cause:** Nothing holds the PID file's lock, but the process it names is alive and something accepts connections on the server's address, so `stop` cannot tell whether that process is the server and does not signal it. A server that an earlier release started with `serve --foreground --write-pid-file` holds no lock on its PID file. + +**Fix:** Check the process with `ps -p `. If it is the server, stop it with `kill `; a server started by this release holds the lock, and `extenddb stop` works with it. + ### `Failed to send SIGTERM to pid : ` **Cause:** `extenddb stop` found a valid PID but could not send SIGTERM. Common causes: permission denied (the extenddb process is owned by a different user) or the process exited between the liveness check and the signal send. diff --git a/tests/test_cli_container_readiness.py b/tests/test_cli_container_readiness.py index 78fe0f3f5..726682889 100644 --- a/tests/test_cli_container_readiness.py +++ b/tests/test_cli_container_readiness.py @@ -568,3 +568,76 @@ def test_write_pid_file_opt_in(self, cli_env, tmp_path): proc.kill() proc.wait(timeout=10) + +class TestStopPidFileLock: + """`stop` signals only the process that holds the PID file's lock. + + A running server holds a lock on its PID file. A file that nothing holds + was left by a server that is gone, and the PID it names may belong to + another process by now. Needs no backend: `stop` runs with no config file + and reads the PID file from the default run directory under HOME. + """ + + @staticmethod + def _stale_pid_file(tmp_path, port: int, pid: int) -> tuple[_IsolatedHome, str]: + """Write `pid` to the default PID file for `port`, with no lock on it.""" + home = _IsolatedHome(tmp_path) + run_dir = os.path.join(home.home, ".extenddb", "run") + os.makedirs(run_dir) + pid_file = os.path.join(run_dir, f"extenddb-{port}.pid") + with open(pid_file, "w") as f: + f.write(f"{pid}\n") + return home, pid_file + + @staticmethod + def _cli(tmp_path, home: _IsolatedHome, *args: str) -> subprocess.CompletedProcess: + env = os.environ.copy() + env.update(home.env) + return subprocess.run( + [EXTENDDB_BINARY, *args], + capture_output=True, text=True, stdin=subprocess.DEVNULL, + env=env, cwd=tmp_path, timeout=30, + ) + + def test_stop_leaves_a_process_that_holds_no_pid_file_lock(self, tmp_path): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + bystander = subprocess.Popen(["sleep", "300"], stdin=subprocess.DEVNULL) + try: + home, pid_file = self._stale_pid_file(tmp_path, port, bystander.pid) + stop = self._cli(tmp_path, home, "stop", "--port", str(port)) + output = stop.stdout + stop.stderr + assert stop.returncode == 0, output + assert "not an extenddb server" in output, output + assert bystander.poll() is None, "`stop` signalled a process that is not the server" + assert not os.path.exists(pid_file), "the stale PID file should be removed" + finally: + bystander.kill() + bystander.wait(timeout=10) + + def test_stop_refuses_an_unlocked_pid_file_while_the_port_is_in_use(self, tmp_path): + """With the port in use, the process could be a server an earlier + release started with --write-pid-file, which holds no lock: `stop` + neither signals it nor removes the file, and exits 1.""" + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + port = listener.getsockname()[1] + bystander = subprocess.Popen(["sleep", "300"], stdin=subprocess.DEVNULL) + try: + home, pid_file = self._stale_pid_file(tmp_path, port, bystander.pid) + stop = self._cli(tmp_path, home, "stop", "--port", str(port)) + output = stop.stdout + stop.stderr + assert stop.returncode == 1, output + assert "cannot tell whether that process is the server" in output, output + assert f"kill {bystander.pid}" in output, output + assert bystander.poll() is None, "`stop` signalled a process it could not identify" + assert os.path.exists(pid_file), "the PID file should be kept" + + status = self._cli(tmp_path, home, "status", "--port", str(port)) + assert "pid unknown" in status.stdout, status.stdout + status.stderr + finally: + listener.close() + bystander.kill() + bystander.wait(timeout=10)