From ef7b2917591c4e7d8446ef8db4fb0c06a5d90282 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:18:15 +0000 Subject: [PATCH] fix(app): stop signals only the process that holds the PID file lock extenddb stop read the PID file, checked that the process existed with kill(pid, 0), and sent it SIGTERM. When a server exited without removing its PID file and the system later gave its PID to another process, stop signalled that process. With a build of main, a PID file naming a sleep process under a fresh HOME made stop print "extenddb stopped" and end the sleep. A running server now holds an exclusive flock on its PID file. In daemon mode daemonize already took that lock on its own descriptor and keeps it until the process exits. In foreground mode with --write-pid-file the server takes it through the new pid_file::hold, which writes the PID to a new file, locks it and renames it into place, so a reader sees either an old file no process holds or the server's file, complete and locked. Because flock locks belong to open file descriptions, hold leaves a file that already names this process and is locked to daemonize, and leaves a file another running process holds alone; that server then serves without a PID file and logs a warning. In a run directory the server cannot write, hold rewrites an existing PID file in place, as earlier releases did, so a file an init system created for the server still works. serve's error path removes the PID file only when it names this process. stop reads the PID and probes the lock through one open of the file, with pid_file::read. It checks the lock before it checks that the process exists, because kill(pid, 0) fails with EPERM for a server another user runs, and after SIGTERM it removes only the file it read. For a file nothing holds: - A file nothing holds whose process is alive is stale when nothing accepts a connection on the port at the bind address the config file and the EXTENDDB__ variables give (an IPv6 address may be in brackets, as serve accepts it), at the loopback addresses for a wildcard one, or at 127.0.0.1. stop says the process is not an extenddb server, removes the file it read (pid_file::remove_stale checks it is still the same file), signals nothing, and exits 0. - When something accepts a connection there, stop exits 1 with a kill hint and keeps the file, because a server that an earlier release started with --foreground --write-pid-file holds no lock. status shows the PID only when the file is held. The troubleshooting guide lists the two new stop messages, including the case an older server bound to an address the config does not name still falls into, and the run-tests restart hint names the kill fallback. Unit tests cover a held file, a stale file replaced by a new one, a writable file in a read-only directory rewritten in place, a free file that names this process, a file daemonize holds for this process, a file another process holds, removal of only the file that was read, a missing file, and the probed addresses. CLI tests write a sleep process's PID to the PID file and check that stop leaves it running and removes the file, and that with the port in use stop exits 1, keeps the file, and status shows no PID; both fail against a build of main. On loopback under an isolated HOME, stop and status worked against a foreground server and a daemon from this change, and stop refused, without signalling it, a foreground server from a build of main on 127.0.0.1 and on ::1. Fixes #710. --- crates/app/src/cmd_status.rs | 29 ++- crates/app/src/cmd_stop.rs | 146 ++++++++++- crates/server/src/lib.rs | 2 + crates/server/src/pid_file.rs | 355 ++++++++++++++++++++++++++ crates/server/src/serve.rs | 90 ++++++- devtools/run-tests | 4 +- docs/troubleshooting.md | 12 + tests/test_cli_container_readiness.py | 73 ++++++ 8 files changed, 677 insertions(+), 34 deletions(-) create mode 100644 crates/server/src/pid_file.rs diff --git a/crates/app/src/cmd_status.rs b/crates/app/src/cmd_status.rs index e48cd9cad..1389f09fb 100755 --- a/crates/app/src/cmd_status.rs +++ b/crates/app/src/cmd_status.rs @@ -46,17 +46,8 @@ pub fn run(args: &StatusArgs) { |_| extenddb_config::pid_file_path_default(port), |c| extenddb_config::pid_file_path(&config::expand_tilde(&c.server.run_dir), port), ); - let pid_label = std::fs::read_to_string(&pid_file) - .ok() - .and_then(|contents| { - let pid = contents.trim().to_owned(); - let pid_num: i32 = pid.parse().ok()?; - if crate::util::is_process_alive(pid_num) { - Some(pid) - } else { - None - } - }); + let pid_label = server_pid(&pid_file) + .filter(|pid| pid.parse::().is_ok_and(crate::util::is_process_alive)); match pid_label { Some(pid) => println!("extenddb is running on port {port} (pid {pid})"), None => println!("extenddb is running on port {port} (pid unknown)"), @@ -71,6 +62,22 @@ pub fn run(args: &StatusArgs) { } } +/// The PID in the PID file, when a running server holds the file. A file +/// nothing holds names a PID the system may have given to another process. +#[cfg(unix)] +fn server_pid(path: &std::path::Path) -> Option { + let contents = extenddb_server::pid_file::read(path).ok()?; + contents.held.then(|| contents.text.trim().to_owned()) +} + +/// Non-unix: PID files only exist in daemon mode, which is unix-only. +#[cfg(not(unix))] +fn server_pid(path: &std::path::Path) -> Option { + std::fs::read_to_string(path) + .ok() + .map(|s| s.trim().to_owned()) +} + /// Determine the port to check: CLI override > config file > default. fn resolve_port(args: &StatusArgs) -> anyhow::Result { if let Some(port) = args.port { diff --git a/crates/app/src/cmd_stop.rs b/crates/app/src/cmd_stop.rs index 5fa856d72..c5851439c 100755 --- a/crates/app/src/cmd_stop.rs +++ b/crates/app/src/cmd_stop.rs @@ -3,8 +3,9 @@ //! `extenddb stop` — stop the running extenddb server. //! -//! Reads the PID file, sends SIGTERM, polls until the process exits (or -//! times out), and cleans up the stale PID file. +//! Reads the PID file, checks that a running server holds it, sends SIGTERM, +//! polls until the process exits (or times out), and cleans up the stale PID +//! file. #[cfg(unix)] use std::thread; @@ -66,8 +67,11 @@ pub fn run(args: &StopArgs) { None => extenddb_config::pid_file_path_default(port), }; - let pid_str = match std::fs::read_to_string(&pid_file) { - Ok(s) => s.trim().to_owned(), + // The PID and whether a running server holds the file, from one open of + // it, so a server that replaces the file meanwhile cannot pair its lock + // with the stale PID. + let contents = match extenddb_server::pid_file::read(&pid_file) { + Ok(contents) => contents, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { // `serve --foreground` writes no PID file, so a missing file does // not mean nothing is running. Probe the port before saying so. @@ -94,6 +98,7 @@ pub fn run(args: &StopArgs) { std::process::exit(1); } }; + let pid_str = contents.text.trim(); let pid: i32 = if let Ok(p) = pid_str.parse() { p @@ -102,12 +107,51 @@ pub fn run(args: &StopArgs) { std::process::exit(1); }; - // Check if the process is alive before sending SIGTERM. - if !is_process_alive(pid) { - eprintln!("Process {pid} is not running. Cleaning up stale PID file."); - let _ = std::fs::remove_file(&pid_file); - // Exit 0: the user's intent ("stop extenddb") is satisfied — the server - // is not running. This makes scripting easier (e.g. `extenddb stop && ...`). + // A running server holds a lock on its PID file. When nothing holds it, the + // PID is stale: the server is gone and the system may have given the PID to + // another process, which must not be signalled. The lock is checked first + // because `is_process_alive` reports a server another user runs as gone. + if !contents.held { + if !is_process_alive(pid) { + eprintln!("Process {pid} is not running. Cleaning up stale PID file."); + let _ = extenddb_server::pid_file::remove_stale(&pid_file, &contents); + // Exit 0: the user's intent ("stop extenddb") is satisfied, since + // the server is not running. This makes scripting easier (e.g. + // `extenddb stop && ...`). + return; + } + // The address `serve` would bind with the same config file and + // `EXTENDDB__` variables, including when there is no file. + let bind_addr = app_config + .as_ref() + .map(|c| c.server.bind_addr.clone()) + .or_else(|| { + extenddb_config::load_builtin_defaults() + .ok() + .map(|c| c.server.bind_addr) + }) + .unwrap_or_else(|| extenddb_config::ServerConfig::default().bind_addr); + let probed = listen_probe_addrs(&bind_addr, port); + if let Some(addr) = probed.iter().find(|a| is_listening(a)) { + eprintln!( + "PID file {} names process {pid}, but no extenddb server holds the file, so \ + `stop` cannot tell whether that process is the server listening on {addr}, \ + and does not signal it.\n\ + A server that an earlier extenddb release started with `serve --foreground \ + --write-pid-file` holds no lock on its PID file. If process {pid} is that \ + server, stop it with: kill {pid}", + pid_file.display(), + ); + std::process::exit(1); + } + let probed: Vec = probed.iter().map(ToString::to_string).collect(); + eprintln!( + "PID file {} names process {pid}, which is not an extenddb server: no server \ + holds the file and nothing listens on {}. Cleaning up stale PID file.", + pid_file.display(), + probed.join(", "), + ); + let _ = extenddb_server::pid_file::remove_stale(&pid_file, &contents); return; } @@ -125,8 +169,9 @@ pub fn run(args: &StopArgs) { let deadline = Instant::now() + SHUTDOWN_TIMEOUT; loop { if !is_process_alive(pid) { - // Process exited — clean up PID file if the server didn't already. - let _ = std::fs::remove_file(&pid_file); + // Process exited: clean up its PID file if the server didn't + // already, but not a file a new server has put in its place. + let _ = extenddb_server::pid_file::remove_stale(&pid_file, &contents); println!("extenddb stopped (pid {pid})"); return; } @@ -140,3 +185,80 @@ pub fn run(args: &StopArgs) { thread::sleep(POLL_INTERVAL); } } + +/// The addresses a server on `port` configured with `bind_addr` answers on: +/// that address, both loopback addresses for a wildcard one, the addresses a +/// host name resolves to, and 127.0.0.1 in every case. +#[cfg(unix)] +fn listen_probe_addrs(bind_addr: &str, port: u16) -> Vec { + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr, ToSocketAddrs}; + + // serve binds `{bind_addr}:{port}`, so it also accepts an IPv6 address in + // brackets. + let host = bind_addr + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(bind_addr); + let mut addrs = vec![SocketAddr::from((Ipv4Addr::LOCALHOST, port))]; + match host.parse::() { + Ok(ip) if ip.is_unspecified() => addrs.push(SocketAddr::from((Ipv6Addr::LOCALHOST, port))), + Ok(ip) => addrs.push(SocketAddr::new(ip, port)), + Err(_) => addrs.extend((host, port).to_socket_addrs().into_iter().flatten()), + } + addrs.sort_unstable(); + addrs.dedup(); + addrs +} + +/// Whether something accepts a TCP connection on `addr`. +#[cfg(unix)] +fn is_listening(addr: &std::net::SocketAddr) -> bool { + std::net::TcpStream::connect_timeout(addr, Duration::from_millis(500)).is_ok() +} + +#[cfg(all(test, unix))] +mod tests { + use std::net::SocketAddr; + + use super::listen_probe_addrs; + + fn addrs(list: &[&str]) -> Vec { + let mut v: Vec = list.iter().map(|a| a.parse().unwrap()).collect(); + v.sort_unstable(); + v + } + + #[test] + fn a_specific_address_is_probed_with_loopback() { + assert_eq!( + listen_probe_addrs("10.1.2.3", 18443), + addrs(&["10.1.2.3:18443", "127.0.0.1:18443"]) + ); + for ipv6 in ["::1", "[::1]"] { + assert_eq!( + listen_probe_addrs(ipv6, 18443), + addrs(&["[::1]:18443", "127.0.0.1:18443"]), + "{ipv6}" + ); + } + } + + #[test] + fn a_wildcard_address_is_probed_on_both_loopbacks() { + for wildcard in ["0.0.0.0", "::", "[::]"] { + assert_eq!( + listen_probe_addrs(wildcard, 18443), + addrs(&["127.0.0.1:18443", "[::1]:18443"]), + "{wildcard}" + ); + } + } + + #[test] + fn the_default_bind_address_is_probed_once() { + assert_eq!( + listen_probe_addrs("127.0.0.1", 18443), + addrs(&["127.0.0.1:18443"]) + ); + } +} diff --git a/crates/server/src/lib.rs b/crates/server/src/lib.rs index f608c74d7..76a785a65 100755 --- a/crates/server/src/lib.rs +++ b/crates/server/src/lib.rs @@ -19,6 +19,8 @@ pub mod key_info_cache; mod leading_keys; pub mod management; mod metrics_endpoint; +#[cfg(unix)] +pub mod pid_file; pub mod rate_limit; mod registration; mod request_helpers; diff --git a/crates/server/src/pid_file.rs b/crates/server/src/pid_file.rs new file mode 100644 index 000000000..eb1083b58 --- /dev/null +++ b/crates/server/src/pid_file.rs @@ -0,0 +1,355 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The PID file that `extenddb status` and `extenddb stop` read. +//! +//! A running server holds an exclusive `flock` on its PID file: `daemonize` +//! takes it in daemon mode, and [`hold`] takes it in foreground mode. A reader +//! that finds the lock free knows that no server owns the file, so the PID in +//! it is stale even when a process with that PID is alive, for example after +//! the server crashed and the system gave its PID to another process. +//! +//! `flock` locks belong to open file descriptions, not to processes, so two +//! descriptors of the same file conflict even inside one process. [`read`] +//! takes the PID and the lock state from one descriptor, and [`hold`] writes a +//! new file and renames it into place, so a reader sees either an old file no +//! process holds or the server's file, complete and held. In a run directory +//! the server cannot write, [`hold`] rewrites an existing file in place. + +use std::fs::{File, OpenOptions}; +use std::io::{self, Read, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::MetadataExt; +use std::path::Path; + +/// What [`hold`] did with the PID file. +#[derive(Debug)] +pub enum Hold { + /// This process wrote the file and holds its lock for as long as the + /// `File` stays open. + Taken(File), + /// The file already names this process and is locked: in daemon mode + /// `daemonize` wrote it and keeps the lock on its own descriptor. + Daemonized, + /// Another running process holds the file, for example a server on the + /// same port bound to another address. The file is left as it is; the + /// value is the PID it names. + HeldByOther(String), +} + +/// A PID file as one open of it saw it. +#[derive(Debug)] +pub struct Contents { + /// The file's text, normally a PID and a newline. + pub text: String, + /// Whether a process held the file's lock, that is, whether a running + /// server owns it. + pub held: bool, + dev: u64, + ino: u64, +} + +/// Write this process's PID to `path` and hold an exclusive lock on the file. +/// +/// The PID goes to a new file that is renamed into place. When the run +/// directory cannot be written but the PID file can, as when an init system +/// created the file for the server, the file is rewritten in place, as +/// earlier releases did. +/// +/// # Errors +/// +/// Returns the I/O error when the existing file cannot be read or the new +/// file cannot be created, locked, written, or renamed into place. +pub fn hold(path: &Path) -> io::Result { + let pid = std::process::id().to_string(); + match read(path) { + Ok(existing) if existing.held => { + let named = existing.text.trim(); + return Ok(if named == pid { + Hold::Daemonized + } else { + Hold::HeldByOther(named.to_owned()) + }); + } + Ok(_) => {} + Err(e) if e.kind() == io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + let tmp = path.with_extension(format!("pid.{pid}.tmp")); + let write = || -> io::Result { + let mut file = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&tmp)?; + try_lock(&file, libc::LOCK_EX)?; + file.write_all(format!("{pid}\n").as_bytes())?; + std::fs::rename(&tmp, path)?; + Ok(file) + }; + match write() { + Ok(file) => Ok(Hold::Taken(file)), + Err(e) => { + let _ = std::fs::remove_file(&tmp); + if e.kind() == io::ErrorKind::PermissionDenied && path.exists() { + return rewrite(path, &pid).map(Hold::Taken); + } + Err(e) + } + } +} + +/// Lock the existing file at `path` and replace its text with `pid`. A reader +/// that read the old text can then remove the file this server holds, which +/// the rename avoids, but only a process that can write the directory can +/// remove a file in it. +fn rewrite(path: &Path, pid: &str) -> io::Result { + let mut file = OpenOptions::new().write(true).open(path)?; + try_lock(&file, libc::LOCK_EX)?; + file.set_len(0)?; + file.write_all(format!("{pid}\n").as_bytes())?; + Ok(file) +} + +/// Read the PID file at `path` and probe its lock through the same open. +/// +/// # Errors +/// +/// Returns the I/O error when the file cannot be opened, read, or probed; a +/// missing file is `NotFound`. +pub fn read(path: &Path) -> io::Result { + let mut file = File::open(path)?; + let held = match try_lock(&file, libc::LOCK_SH) { + Ok(()) => { + // Free, so no server owns it. Release the shared lock at once. A + // foreground server locks its new file before renaming it into + // place, so it never meets this lock. `daemonize` opens the file + // at its path and then locks it without waiting, so a daemon that + // starts in the two calls this lock lasts fails to start. + unlock(&file)?; + false + } + Err(e) if e.kind() == io::ErrorKind::WouldBlock => true, + Err(e) => return Err(e), + }; + let mut text = String::new(); + file.read_to_string(&mut text)?; + let meta = file.metadata()?; + Ok(Contents { + text, + held, + dev: meta.dev(), + ino: meta.ino(), + }) +} + +/// Remove the stale PID file at `path`, unless the path now names another +/// file than the one `contents` was read from, as it does once a new server +/// has renamed its own file into place. +/// +/// # Errors +/// +/// Returns the I/O error of the removal; a file already gone is not an error. +pub fn remove_stale(path: &Path, contents: &Contents) -> io::Result<()> { + match std::fs::metadata(path) { + Ok(meta) if meta.dev() == contents.dev && meta.ino() == contents.ino => { + match std::fs::remove_file(path) { + Err(e) if e.kind() != io::ErrorKind::NotFound => Err(e), + _ => Ok(()), + } + } + Ok(_) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } +} + +fn try_lock(file: &File, operation: libc::c_int) -> io::Result<()> { + // SAFETY: `flock` reads only the descriptor, which `file` keeps open for + // the duration of the call. + if unsafe { libc::flock(file.as_raw_fd(), operation | libc::LOCK_NB) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +fn unlock(file: &File) -> io::Result<()> { + // SAFETY: as in `try_lock`. + if unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_UN) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} + +#[cfg(test)] +mod tests { + use std::fs::File; + use std::os::fd::AsRawFd; + use std::os::unix::fs::MetadataExt; + use std::path::PathBuf; + + use super::{Hold, hold, read, remove_stale}; + + fn scratch(name: &str) -> (PathBuf, PathBuf) { + let dir = std::env::temp_dir().join(format!( + "extenddb-pid-file-{name}-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock") + .as_nanos() + )); + std::fs::create_dir(&dir).expect("scratch dir"); + let path = dir.join("extenddb-18443.pid"); + (dir, path) + } + + fn own_pid_line() -> String { + format!("{}\n", std::process::id()) + } + + /// A file locked on its own descriptor, as `daemonize` or another server + /// leaves it. + fn locked_file(path: &PathBuf, text: &str) -> File { + let file = File::create(path).unwrap(); + std::fs::write(path, text).unwrap(); + // SAFETY: `file` keeps the descriptor open for the call. + assert_eq!( + unsafe { libc::flock(file.as_raw_fd(), libc::LOCK_EX | libc::LOCK_NB) }, + 0 + ); + file + } + + fn taken(result: Hold) -> File { + match result { + Hold::Taken(file) => file, + other => panic!("expected Taken, got {other:?}"), + } + } + + /// The file names this process while it is held, and a reader sees the + /// lock until the holder is dropped. + #[test] + fn a_held_pid_file_is_seen_as_held_until_it_is_dropped() { + let (dir, path) = scratch("held"); + let held = taken(hold(&path).expect("hold")); + let seen = read(&path).unwrap(); + assert_eq!(seen.text, own_pid_line()); + assert!(seen.held); + drop(held); + assert!(!read(&path).unwrap().held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A PID file left by a server that is gone holds no lock, whatever PID it + /// names, and a new server replaces it with a new file. + #[test] + fn a_stale_pid_file_is_free_and_is_replaced() { + let (dir, path) = scratch("stale"); + std::fs::write(&path, "1\n").unwrap(); + let old_ino = std::fs::metadata(&path).unwrap().ino(); + assert!(!read(&path).unwrap().held); + let held = taken(hold(&path).expect("hold")); + assert_eq!(std::fs::read_to_string(&path).unwrap(), own_pid_line()); + assert_ne!(std::fs::metadata(&path).unwrap().ino(), old_ino); + assert!(read(&path).unwrap().held); + drop(held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A PID file the server can write, in a run directory it cannot, is + /// rewritten in place and held. + #[test] + fn a_writable_file_in_a_read_only_directory_is_rewritten_in_place() { + use std::os::unix::fs::PermissionsExt; + + // Root writes any directory, so the rename path would run instead. + // SAFETY: `geteuid` takes no arguments and cannot fail. + if unsafe { libc::geteuid() } == 0 { + return; + } + let (dir, path) = scratch("read-only-dir"); + std::fs::write(&path, "1\n").unwrap(); + let ino = std::fs::metadata(&path).unwrap().ino(); + let mode = |m| std::fs::Permissions::from_mode(m); + std::fs::set_permissions(&dir, mode(0o555)).unwrap(); + let result = hold(&path); + std::fs::set_permissions(&dir, mode(0o755)).unwrap(); + let held = taken(result.expect("hold")); + assert_eq!(std::fs::read_to_string(&path).unwrap(), own_pid_line()); + assert_eq!(std::fs::metadata(&path).unwrap().ino(), ino); + assert!(read(&path).unwrap().held); + drop(held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A free file that names this process by coincidence, after the system + /// reused the PID of an earlier server, is not mistaken for daemonize's. + #[test] + fn a_free_file_naming_this_process_is_taken_over() { + let (dir, path) = scratch("own-stale"); + std::fs::write(&path, own_pid_line()).unwrap(); + let held = taken(hold(&path).expect("hold")); + assert!(read(&path).unwrap().held); + drop(held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// In daemon mode `daemonize` has already written this PID and keeps the + /// lock on its own descriptor; `hold` leaves both alone. + #[test] + fn a_file_daemonize_holds_for_this_process_is_left_to_it() { + let (dir, path) = scratch("daemon"); + let daemon = locked_file(&path, &own_pid_line()); + assert!(matches!(hold(&path).expect("hold"), Hold::Daemonized)); + assert!(read(&path).unwrap().held); + drop(daemon); + assert!(!read(&path).unwrap().held); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A file another running process holds is not touched. + #[test] + fn a_file_another_process_holds_is_left_alone() { + let (dir, path) = scratch("other"); + let other = locked_file(&path, "424242\n"); + match hold(&path).expect("hold") { + Hold::HeldByOther(pid) => assert_eq!(pid, "424242"), + other => panic!("expected HeldByOther, got {other:?}"), + } + assert_eq!(std::fs::read_to_string(&path).unwrap(), "424242\n"); + drop(other); + std::fs::remove_dir_all(dir).unwrap(); + } + + /// A stale file is removed, but not a new file that a server renamed into + /// its place after the stale one was read. + #[test] + fn remove_stale_removes_only_the_file_that_was_read() { + let (dir, path) = scratch("remove"); + std::fs::write(&path, "1\n").unwrap(); + let stale = read(&path).unwrap(); + let held = taken(hold(&path).expect("hold")); + remove_stale(&path, &stale).unwrap(); + assert!(path.exists(), "the new server's file must stay"); + drop(held); + let stale = read(&path).unwrap(); + remove_stale(&path, &stale).unwrap(); + assert!(!path.exists()); + remove_stale(&path, &stale).unwrap(); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn a_missing_pid_file_is_not_found() { + let (dir, path) = scratch("missing"); + assert_eq!( + read(&path).unwrap_err().kind(), + std::io::ErrorKind::NotFound + ); + std::fs::remove_dir_all(dir).unwrap(); + } +} diff --git a/crates/server/src/serve.rs b/crates/server/src/serve.rs index 37cc11f0d..682b36e47 100644 --- a/crates/server/src/serve.rs +++ b/crates/server/src/serve.rs @@ -221,7 +221,12 @@ pub async fn serve(params: ServeParams) -> anyhow::Result<()> { let log_target = params.log_target; let result = serve_inner(params, port).await; if let Err(ref e) = result { - if let Some(ref path) = pid_path { + // Only this process's own file: another running server may hold the + // path when the two share a port number on different addresses. + if let Some(ref path) = pid_path + && std::fs::read_to_string(path) + .is_ok_and(|s| s.trim() == std::process::id().to_string()) + { let _ = std::fs::remove_file(path); } // P57 Bug 7: Log fatal errors where the operator will see them. After @@ -237,6 +242,63 @@ pub async fn serve(params: ServeParams) -> anyhow::Result<()> { result } +/// The PID file a server writes and the lock it keeps on it. +struct PidFileGuard { + /// The file this server owns, which `start_server` removes at shutdown. + path: Option, + /// Keeps the lock on the file this process wrote; `None` when there is no + /// file, or when `daemonize` holds the lock. + lock: Option, + /// The path and PID of a file another running process holds instead. + held_by_other: Option, +} + +#[cfg(unix)] +fn take_pid_file(pid_file: Option) -> anyhow::Result { + use crate::pid_file::Hold; + + let Some(path) = pid_file else { + return Ok(PidFileGuard { + path: None, + lock: None, + held_by_other: None, + }); + }; + let hold = crate::pid_file::hold(&path) + .map_err(|e| anyhow::anyhow!("Failed to write PID file {}: {e}", path.display()))?; + Ok(match hold { + Hold::Taken(file) => PidFileGuard { + path: Some(path), + lock: Some(file), + held_by_other: None, + }, + Hold::Daemonized => PidFileGuard { + path: Some(path), + lock: None, + held_by_other: None, + }, + Hold::HeldByOther(pid) => PidFileGuard { + held_by_other: Some(format!("{} (pid {pid})", path.display())), + path: None, + lock: None, + }, + }) +} + +/// Non-unix: no locks, and `stop` and `status` do not read PID files. +#[cfg(not(unix))] +fn take_pid_file(pid_file: Option) -> anyhow::Result { + if let Some(ref path) = pid_file { + std::fs::write(path, format!("{}\n", std::process::id())) + .map_err(|e| anyhow::anyhow!("Failed to write PID file {}: {e}", path.display()))?; + } + Ok(PidFileGuard { + path: pid_file, + lock: None, + held_by_other: None, + }) +} + /// Inner serve function — separated so [`serve`] can clean up the PID file on /// any error path. async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { @@ -252,15 +314,16 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { let catalog_version = extenddb_storage::migrations::binary_catalog_label(); // Write the PID file, when the caller asked for one, so `extenddb status`, - // `extenddb stop`, and `start_server`'s graceful shutdown cleanup work. When - // the caller daemonized, the grandchild PID that `daemonize` wrote is the - // same value as `std::process::id()` post-fork, so this is a consistent - // rewrite rather than a conflicting one. A supervised deployment passes - // `None` and needs no writable run directory at all. - if let Some(ref path) = pid_file { - std::fs::write(path, format!("{}\n", std::process::id())) - .map_err(|e| anyhow::anyhow!("Failed to write PID file {}: {e}", path.display()))?; - } + // `extenddb stop`, and `start_server`'s graceful shutdown cleanup work. A + // supervised deployment passes `None` and needs no writable run directory + // at all. On unix the server holds a lock on the file until it exits, which + // is how `status` and `stop` tell its PID from a stale one; in daemon mode + // `daemonize` has already written the same PID and holds that lock. + let PidFileGuard { + path: pid_file, + lock: _pid_file_lock, + held_by_other, + } = take_pid_file(pid_file)?; // Init logging (REQ-LOG-003, REQ-LOG-006) — the caller chose the target via // [`LogTarget`]; a supervised/container deployment picks stderr so the @@ -329,6 +392,13 @@ async fn serve_inner(params: ServeParams, port: u16) -> anyhow::Result<()> { .try_init() .map_err(|e| anyhow::anyhow!("Failed to initialize tracing: {e}"))?; + if let Some(other) = held_by_other { + tracing::warn!( + "PID file {other} belongs to another running server; serving without a PID file, \ + so `extenddb status` and `extenddb stop` find that server, not this one" + ); + } + for warning in app_config.storage.startup_warnings() { tracing::warn!("{}", warning.log_message()); } diff --git a/devtools/run-tests b/devtools/run-tests index 421a97b3a..28b0f7d81 100755 --- a/devtools/run-tests +++ b/devtools/run-tests @@ -366,7 +366,9 @@ if $NEEDS_INTEGRATION && [[ "$TARGET" != "real-dynamodb" ]]; then if ! "$BINARY" stop --config "$CONFIG_FOR_SETTINGS"; then echo " ✗ Could not stop the running server to apply the config change." echo " If it was started with 'serve --foreground', restart it with" - echo " --write-pid-file so 'stop' can signal it." + echo " --write-pid-file so 'stop' can signal it. A server that an earlier" + echo " release started with --write-pid-file holds no lock on its PID file;" + echo " stop it with the kill command 'stop' printed above." exit 1 fi sleep 1 diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index f22a2c4d1..e6db9cbdd 100755 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -234,6 +234,18 @@ journalctl -t extenddb -f **Fix:** Delete the PID file manually and use `extenddb status` or `ps` to find the extenddb process. Stop it with `kill `. +### `PID file names process , which is not an extenddb server` + +**Cause:** A running server holds a lock on its PID file, and nothing holds this one. The process it names is alive, but nothing accepts connections on the port at the bind address `stop` reads from the config file and the `EXTENDDB__` variables, or at 127.0.0.1. `stop` takes the process for one the system gave the PID to after the server exited without removing its file, for example after a crash. It does not signal it, removes the stale file, and exits 0. + +**Fix:** If no server should be running, none. A server that an earlier release started with `serve --foreground --write-pid-file`, bound to an address the config `stop` read does not name, also holds no lock: if `extenddb serve` then fails to bind the port, find that server with `ss -ltnp` and stop it with `kill `. + +### `PID file names process , but no extenddb server holds the file` + +**Cause:** Nothing holds the PID file's lock, but the process it names is alive and something accepts connections on the server's address, so `stop` cannot tell whether that process is the server and does not signal it. A server that an earlier release started with `serve --foreground --write-pid-file` holds no lock on its PID file. + +**Fix:** Check the process with `ps -p `. If it is the server, stop it with `kill `; a server started by this release holds the lock, and `extenddb stop` works with it. + ### `Failed to send SIGTERM to pid : ` **Cause:** `extenddb stop` found a valid PID but could not send SIGTERM. Common causes: permission denied (the extenddb process is owned by a different user) or the process exited between the liveness check and the signal send. diff --git a/tests/test_cli_container_readiness.py b/tests/test_cli_container_readiness.py index 78fe0f3f5..726682889 100644 --- a/tests/test_cli_container_readiness.py +++ b/tests/test_cli_container_readiness.py @@ -568,3 +568,76 @@ def test_write_pid_file_opt_in(self, cli_env, tmp_path): proc.kill() proc.wait(timeout=10) + +class TestStopPidFileLock: + """`stop` signals only the process that holds the PID file's lock. + + A running server holds a lock on its PID file. A file that nothing holds + was left by a server that is gone, and the PID it names may belong to + another process by now. Needs no backend: `stop` runs with no config file + and reads the PID file from the default run directory under HOME. + """ + + @staticmethod + def _stale_pid_file(tmp_path, port: int, pid: int) -> tuple[_IsolatedHome, str]: + """Write `pid` to the default PID file for `port`, with no lock on it.""" + home = _IsolatedHome(tmp_path) + run_dir = os.path.join(home.home, ".extenddb", "run") + os.makedirs(run_dir) + pid_file = os.path.join(run_dir, f"extenddb-{port}.pid") + with open(pid_file, "w") as f: + f.write(f"{pid}\n") + return home, pid_file + + @staticmethod + def _cli(tmp_path, home: _IsolatedHome, *args: str) -> subprocess.CompletedProcess: + env = os.environ.copy() + env.update(home.env) + return subprocess.run( + [EXTENDDB_BINARY, *args], + capture_output=True, text=True, stdin=subprocess.DEVNULL, + env=env, cwd=tmp_path, timeout=30, + ) + + def test_stop_leaves_a_process_that_holds_no_pid_file_lock(self, tmp_path): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + port = s.getsockname()[1] + bystander = subprocess.Popen(["sleep", "300"], stdin=subprocess.DEVNULL) + try: + home, pid_file = self._stale_pid_file(tmp_path, port, bystander.pid) + stop = self._cli(tmp_path, home, "stop", "--port", str(port)) + output = stop.stdout + stop.stderr + assert stop.returncode == 0, output + assert "not an extenddb server" in output, output + assert bystander.poll() is None, "`stop` signalled a process that is not the server" + assert not os.path.exists(pid_file), "the stale PID file should be removed" + finally: + bystander.kill() + bystander.wait(timeout=10) + + def test_stop_refuses_an_unlocked_pid_file_while_the_port_is_in_use(self, tmp_path): + """With the port in use, the process could be a server an earlier + release started with --write-pid-file, which holds no lock: `stop` + neither signals it nor removes the file, and exits 1.""" + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(("127.0.0.1", 0)) + listener.listen(1) + port = listener.getsockname()[1] + bystander = subprocess.Popen(["sleep", "300"], stdin=subprocess.DEVNULL) + try: + home, pid_file = self._stale_pid_file(tmp_path, port, bystander.pid) + stop = self._cli(tmp_path, home, "stop", "--port", str(port)) + output = stop.stdout + stop.stderr + assert stop.returncode == 1, output + assert "cannot tell whether that process is the server" in output, output + assert f"kill {bystander.pid}" in output, output + assert bystander.poll() is None, "`stop` signalled a process it could not identify" + assert os.path.exists(pid_file), "the PID file should be kept" + + status = self._cli(tmp_path, home, "status", "--port", str(port)) + assert "pid unknown" in status.stdout, status.stdout + status.stderr + finally: + listener.close() + bystander.kill() + bystander.wait(timeout=10)