From 8e1a2dc0698cf628eac9bcca951bb6686d184a59 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 01:16:29 +0000 Subject: [PATCH 1/4] Stop abandoned admin requests from holding database connections When a client gives up on an admin request, the server drops it but Postgres keeps running its query, and sqlx holds the connection until the query ends. If the query then ended in an error, such as a statement timeout, sqlx 0.9's release check reported that error as a broken connection and closed it with a hard close, which hangs on a TLS connection: the pool lost that slot for good. A slow read polled from an open dashboard could exhaust the pool. awa serve now builds its pool with awa_ui::admin_pool_options and admin_connect_options: - a statement timeout as a startup option (60 s, --statement-timeout / AWA_STATEMENT_TIMEOUT, 0 to leave the server's setting), so an abandoned query ends and RESET returns to the bound; - a release hook that reads an abandoned query's result, treats its database error as that query's, and confirms the connection with a second ping, so the pool reuses it instead of closing it; - client_connection_check_interval = 5s on Postgres 14+, so a query stops when its connection closes, for example when awa serve restarts. pool_limits_test drops a request mid-query against a one-connection pool and checks that the next request gets the same backend within the timeout; without the release hook it times out acquiring. --- CHANGELOG.md | 12 +++++ awa-cli/src/main.rs | 14 ++++- awa-ui/src/lib.rs | 4 ++ awa-ui/src/pool.rs | 81 ++++++++++++++++++++++++++++ awa-ui/tests/pool_limits_test.rs | 91 ++++++++++++++++++++++++++++++++ docs/configuration.md | 5 ++ docs/ui-design.md | 12 +++++ 7 files changed, 217 insertions(+), 2 deletions(-) create mode 100644 awa-ui/src/pool.rs create mode 100644 awa-ui/tests/pool_limits_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index a9819d5a1..2f49a01ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,18 @@ Notable changes between releases. Detailed migration notes for storage transitio ### Fixed +- **A request the admin UI's client abandons no longer holds a database + connection for good.** When a browser closes a tab or a proxy times out, + the server drops the request but Postgres keeps running its query. If that + query then ended in an error, such as a statement timeout, sqlx 0.9 tried + to close the connection, and closing a TLS connection mid-stream hangs, so + the pool lost that slot permanently; a slow dashboard read polled often + enough could exhaust the pool. `awa serve` now sets a 60-second statement + timeout (`--statement-timeout`, `AWA_STATEMENT_TIMEOUT`), recovers such + connections when they are released, and on Postgres 14+ sets + `client_connection_check_interval` so a query stops when its connection + closes. Applications embedding `awa_ui::router` can build their pool with + the new `awa_ui::admin_pool_options` and `admin_connect_options`. - **Queue overviews on queue storage no longer read retained history.** `admin::queue_overviews`, behind the admin UI's queues and dashboard pages and `awa queue stats`, counted every current job through `terminal_jobs`, diff --git a/awa-cli/src/main.rs b/awa-cli/src/main.rs index f89003114..405fc3023 100644 --- a/awa-cli/src/main.rs +++ b/awa-cli/src/main.rs @@ -126,6 +126,11 @@ enum Commands { /// Seconds to wait when acquiring a connection #[arg(long, default_value = "10", env = "AWA_POOL_ACQUIRE_TIMEOUT")] pool_acquire_timeout: u64, + /// Seconds any one database statement from the UI may run; 0 leaves + /// the server's setting. Bounds queries a closed or timed-out request + /// leaves running, and the pool connections they hold. + #[arg(long, default_value = "60", env = "AWA_STATEMENT_TIMEOUT")] + statement_timeout: u64, /// Cache TTL for dashboard queries in seconds #[arg(long, default_value = "5", env = "AWA_CACHE_TTL")] cache_ttl: u64, @@ -1189,6 +1194,7 @@ async fn run() -> Result<(), Box> { pool_idle_timeout, pool_max_lifetime, pool_acquire_timeout, + statement_timeout, cache_ttl, callback_hmac_secret, read_only, @@ -1212,13 +1218,17 @@ async fn run() -> Result<(), Box> { .collect::, _>>()?, }; let db_url = resolve_db()?; - let pool = PgPoolOptions::new() + let connect_options = awa_ui::admin_connect_options( + db_url.parse()?, + Some(Duration::from_secs(statement_timeout)), + ); + let pool = awa_ui::admin_pool_options(PgPoolOptions::new()) .max_connections(pool_max) .min_connections(pool_min) .idle_timeout(Duration::from_secs(pool_idle_timeout)) .max_lifetime(Duration::from_secs(pool_max_lifetime)) .acquire_timeout(Duration::from_secs(pool_acquire_timeout)) - .connect(&db_url) + .connect_with(connect_options) .await?; let cache_duration = Duration::from_secs(cache_ttl); diff --git a/awa-ui/src/lib.rs b/awa-ui/src/lib.rs index 6eae15bf7..b4de9e596 100644 --- a/awa-ui/src/lib.rs +++ b/awa-ui/src/lib.rs @@ -2,11 +2,15 @@ pub mod cache; pub mod callback_router; pub mod error; pub mod handlers; +pub mod pool; pub mod state; pub use callback_router::{ callback_router, CallbackAuth, CallbackReceiverConfig, CallbackRouterBuildError, }; +pub use pool::{ + admin_connect_options, admin_pool_options, configure_admin_connection, recover_admin_connection, +}; use std::time::Duration; diff --git a/awa-ui/src/pool.rs b/awa-ui/src/pool.rs new file mode 100644 index 000000000..d5a650dbb --- /dev/null +++ b/awa-ui/src/pool.rs @@ -0,0 +1,81 @@ +//! Session limits for the connection pool that serves the admin UI. +//! +//! When a client gives up on a request (a closed tab, a proxy timeout, a +//! poll that outlives its interval), the handler is dropped but its query +//! keeps running on Postgres: nothing cancels it. sqlx then waits for that +//! query to finish before the connection can rejoin the pool. A slow admin +//! read polled from an open dashboard therefore stacks one running query +//! and one held connection per poll. +//! +//! These settings bound both: a statement timeout ends the abandoned query, +//! and the release hook returns its connection to the pool afterwards. +//! Without that hook, sqlx 0.9 treats the abandoned query's error as a broken +//! connection and closes it, and closing a TLS connection mid-stream hangs, +//! so the pool slot would never be freed. + +use std::time::Duration; + +use sqlx::postgres::{PgConnectOptions, PgConnection, PgPoolOptions}; +use sqlx::Connection; + +/// `client_connection_check_interval` exists from Postgres 14. +const CLIENT_CONNECTION_CHECK_MIN_VERSION: u32 = 140_000; + +/// Add `statement_timeout` to the connection's startup options, so no single +/// admin statement runs longer and `RESET statement_timeout` returns to this +/// value rather than to none. `None` or a zero duration leaves the server's +/// setting in place. +pub fn admin_connect_options( + options: PgConnectOptions, + statement_timeout: Option, +) -> PgConnectOptions { + match statement_timeout.filter(|timeout| !timeout.is_zero()) { + Some(timeout) => { + options.options([("statement_timeout", format!("{}ms", timeout.as_millis()))]) + } + None => options, + } +} + +/// Set the session limits that need a server version check, on each new +/// connection. Use with [`PgPoolOptions::after_connect`], or through +/// [`admin_pool_options`]. +pub async fn configure_admin_connection(conn: &mut PgConnection) -> Result<(), sqlx::Error> { + if conn + .server_version_num() + .is_some_and(|version| version >= CLIENT_CONNECTION_CHECK_MIN_VERSION) + { + // While a query runs, Postgres checks every 5 s whether the client is + // still connected, so a connection the UI server closed, or lost when + // it exited, stops its query instead of running it to the end. + sqlx::query("SET client_connection_check_interval = '5s'") + .execute(&mut *conn) + .await?; + } + Ok(()) +} + +/// Make a released connection reusable after its request was dropped +/// mid-query. Use with [`PgPoolOptions::after_release`], or through +/// [`admin_pool_options`]. +/// +/// The first ping waits for the abandoned query and reports its result: a +/// database error there (the statement timeout, say) belongs to that query, +/// not to the connection, which a second ping confirms. Never returns an +/// error: sqlx closes such a connection with a hard close, which hangs on a +/// TLS connection and leaks its pool slot. `Ok(false)` closes it gracefully. +pub async fn recover_admin_connection(conn: &mut PgConnection) -> Result { + match conn.ping().await { + Ok(()) => Ok(true), + Err(sqlx::Error::Database(_)) => Ok(conn.ping().await.is_ok()), + Err(_) => Ok(false), + } +} + +/// `options` with [`configure_admin_connection`] on every new connection and +/// [`recover_admin_connection`] on every release. +pub fn admin_pool_options(options: PgPoolOptions) -> PgPoolOptions { + options + .after_connect(|conn, _meta| Box::pin(configure_admin_connection(conn))) + .after_release(|conn, _meta| Box::pin(recover_admin_connection(conn))) +} diff --git a/awa-ui/tests/pool_limits_test.rs b/awa-ui/tests/pool_limits_test.rs new file mode 100644 index 000000000..8625721be --- /dev/null +++ b/awa-ui/tests/pool_limits_test.rs @@ -0,0 +1,91 @@ +//! The admin UI pool's session limits bound queries that a dropped request +//! leaves running on Postgres. + +use std::time::{Duration, Instant}; + +use sqlx::postgres::{PgConnectOptions, PgPoolOptions}; +use sqlx::PgPool; + +async fn show(conn: &mut sqlx::PgConnection, setting: &str) -> String { + sqlx::query_scalar(sqlx::AssertSqlSafe(format!("SHOW {setting}"))) + .fetch_one(conn) + .await + .unwrap() +} + +async fn admin_pool(statement_timeout: Duration) -> PgPool { + let options: PgConnectOptions = awa_testing::setup::database_url().parse().unwrap(); + awa_ui::admin_pool_options(PgPoolOptions::new()) + .max_connections(1) + .acquire_timeout(Duration::from_secs(20)) + .connect_with(awa_ui::admin_connect_options( + options, + Some(statement_timeout), + )) + .await + .unwrap() +} + +#[tokio::test] +async fn statement_timeout_is_the_session_default() { + let pool = admin_pool(Duration::from_secs(2)).await; + let mut conn = pool.acquire().await.unwrap(); + assert_eq!(show(&mut conn, "statement_timeout").await, "2s"); + // A startup option, not a session SET, so RESET returns to it. + sqlx::query("SET statement_timeout = 0") + .execute(&mut *conn) + .await + .unwrap(); + sqlx::query("RESET statement_timeout") + .execute(&mut *conn) + .await + .unwrap(); + assert_eq!(show(&mut conn, "statement_timeout").await, "2s"); + + let version: i32 = sqlx::query_scalar("SELECT current_setting('server_version_num')::int") + .fetch_one(&mut *conn) + .await + .unwrap(); + if version >= 140_000 { + assert_eq!( + show(&mut conn, "client_connection_check_interval").await, + "5s" + ); + } +} + +#[tokio::test] +async fn a_dropped_request_frees_its_connection_at_the_statement_timeout() { + let pool = admin_pool(Duration::from_secs(2)).await; + let backend: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&pool) + .await + .unwrap(); + + // The client gives up on a slow read: its future is dropped mid-query. + let abandoned = tokio::time::timeout( + Duration::from_millis(300), + sqlx::query("SELECT pg_sleep(30)").execute(&pool), + ) + .await; + assert!( + abandoned.is_err(), + "the read was still running when dropped" + ); + + // The pool's one connection waits for that query to end before it can + // be reused; the statement timeout ends it, rather than its 30 s sleep. + let started = Instant::now(); + let after: i32 = sqlx::query_scalar("SELECT pg_backend_pid()") + .fetch_one(&pool) + .await + .unwrap(); + // The same connection, recovered rather than closed: closing it after + // the timeout's error is the path that hangs over TLS. + assert_eq!(after, backend); + assert!( + started.elapsed() < Duration::from_secs(10), + "the next request waited {:?} for the abandoned query", + started.elapsed() + ); +} diff --git a/docs/configuration.md b/docs/configuration.md index b80ad3fb7..930bc6bde 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -770,8 +770,13 @@ Every flag has a corresponding `AWA_*` environment variable (shown in `--help`): | `--pool-idle-timeout` | `AWA_POOL_IDLE_TIMEOUT` | `300` | Idle connection timeout (seconds) | | `--pool-max-lifetime` | `AWA_POOL_MAX_LIFETIME` | `1800` | Max connection lifetime (seconds) | | `--pool-acquire-timeout` | `AWA_POOL_ACQUIRE_TIMEOUT` | `10` | Connection acquire timeout (seconds) | +| `--statement-timeout` | `AWA_STATEMENT_TIMEOUT` | `60` | Longest any one UI statement may run (seconds); `0` leaves the server's setting | | `--cache-ttl` | `AWA_CACHE_TTL` | `5` | Dashboard query cache TTL (seconds) | +### Abandoned requests + +When a browser closes a tab or a proxy times out, the server drops the request, but Postgres keeps running its query and the pool keeps that query's connection until it ends. The statement timeout ends it, and the pool then reuses the connection. On Postgres 14 and later the pool also sets `client_connection_check_interval`, so a query stops within 5 s when its connection closes, for example when `awa serve` restarts. Raise `--statement-timeout` if a bulk action from the UI, such as purging a very large DLQ, needs longer. + ### Dashboard cache The cache deduplicates repeated poll requests — multiple browser tabs or rapid refresh cycles within the TTL window hit memory rather than the database. The frontend polling interval is derived from the cache TTL (minimum 5s) and served via `/api/capabilities`, so clients automatically back off to match the server's refresh rate. diff --git a/docs/ui-design.md b/docs/ui-design.md index 17808024c..6d64111aa 100644 --- a/docs/ui-design.md +++ b/docs/ui-design.md @@ -29,6 +29,18 @@ let app = awa_ui::callback_router( .await?; ``` +Give the UI its own pool built with `awa_ui::admin_pool_options` and `awa_ui::admin_connect_options`, as `awa serve` does. They set a statement timeout and a release hook. Without them, a request the client abandons leaves its query running and holds its connection, and over TLS sqlx can lose that connection for good (see [configuration](configuration.md#abandoned-requests)): + +```rust +let pool = awa_ui::admin_pool_options(PgPoolOptions::new()) + .max_connections(10) + .connect_with(awa_ui::admin_connect_options( + database_url.parse()?, + Some(Duration::from_secs(60)), + )) + .await?; +``` + The frontend is a React 19 SPA built with Vite and embedded into the binary via `rust-embed`. The axum handler serves `index.html` for all non-API routes (SPA fallback), and JSON REST endpoints under `/api/`. ### Frontend Stack From c482b22baef103f92b69a2876a442fd8b3bd670a Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 01:24:50 +0000 Subject: [PATCH 2/4] Document the admin statement timeout in the operations skill --- skills/awa-operations/SKILL.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/skills/awa-operations/SKILL.md b/skills/awa-operations/SKILL.md index b36990834..4cfb039b0 100644 --- a/skills/awa-operations/SKILL.md +++ b/skills/awa-operations/SKILL.md @@ -372,6 +372,16 @@ connection. Set `--callback-hmac-secret` to verify callback signatures; for externally reachable callbacks with a private admin surface, run `awa callbacks serve` instead, whose router has no admin routes. +`awa serve` caps every admin query at 60 seconds (`--statement-timeout` / +`AWA_STATEMENT_TIMEOUT`, in seconds; `0` keeps the server's setting). Postgres +keeps running a query after its HTTP request is abandoned, so this cap is what +stops repeated dashboard polls from piling up slow queries. When embedding +`awa_ui::router` in another service, build its pool with +`awa_ui::admin_pool_options(PgPoolOptions::new())` and +`connect_with(awa_ui::admin_connect_options(url.parse()?, Some(timeout)))`. +The release hook recovers connections whose request was dropped mid-query; +without it, a cancelled query can leak the pool slot over TLS. + ## Before You Finish A Change 1. Confirm the resolved target and that you are on the intended context. From 4a95095f78780e676dc755dd7e4ccac3a49328f8 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 01:35:13 +0000 Subject: [PATCH 3/4] Round sub-millisecond admin statement timeouts up to 1ms --- awa-ui/src/pool.rs | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/awa-ui/src/pool.rs b/awa-ui/src/pool.rs index d5a650dbb..02f8fd5b7 100644 --- a/awa-ui/src/pool.rs +++ b/awa-ui/src/pool.rs @@ -31,7 +31,10 @@ pub fn admin_connect_options( ) -> PgConnectOptions { match statement_timeout.filter(|timeout| !timeout.is_zero()) { Some(timeout) => { - options.options([("statement_timeout", format!("{}ms", timeout.as_millis()))]) + // Round up: a sub-millisecond timeout written as `0ms` would + // disable the limit instead of setting one. + let millis = timeout.as_micros().div_ceil(1000); + options.options([("statement_timeout", format!("{millis}ms"))]) } None => options, } @@ -79,3 +82,32 @@ pub fn admin_pool_options(options: PgPoolOptions) -> PgPoolOptions { .after_connect(|conn, _meta| Box::pin(configure_admin_connection(conn))) .after_release(|conn, _meta| Box::pin(recover_admin_connection(conn))) } + +#[cfg(test)] +mod tests { + use super::*; + + fn timeout_option(timeout: Option) -> Option { + admin_connect_options(PgConnectOptions::new(), timeout) + .get_options() + .map(str::to_owned) + } + + #[test] + fn statement_timeout_is_written_in_whole_milliseconds() { + assert_eq!( + timeout_option(Some(Duration::from_secs(60))).as_deref(), + Some("-c statement_timeout=60000ms") + ); + assert_eq!( + timeout_option(Some(Duration::from_micros(500))).as_deref(), + Some("-c statement_timeout=1ms") + ); + assert_eq!( + timeout_option(Some(Duration::from_micros(1_500))).as_deref(), + Some("-c statement_timeout=2ms") + ); + assert_eq!(timeout_option(Some(Duration::ZERO)), None); + assert_eq!(timeout_option(None), None); + } +} From c550b6ac23b0422afd3cfab673f86fca2e182641 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 01:37:10 +0000 Subject: [PATCH 4/4] Round admin statement timeouts up from nanoseconds --- awa-ui/src/pool.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/awa-ui/src/pool.rs b/awa-ui/src/pool.rs index 02f8fd5b7..f8427d504 100644 --- a/awa-ui/src/pool.rs +++ b/awa-ui/src/pool.rs @@ -33,7 +33,7 @@ pub fn admin_connect_options( Some(timeout) => { // Round up: a sub-millisecond timeout written as `0ms` would // disable the limit instead of setting one. - let millis = timeout.as_micros().div_ceil(1000); + let millis = timeout.as_nanos().div_ceil(1_000_000); options.options([("statement_timeout", format!("{millis}ms"))]) } None => options, @@ -99,6 +99,10 @@ mod tests { timeout_option(Some(Duration::from_secs(60))).as_deref(), Some("-c statement_timeout=60000ms") ); + assert_eq!( + timeout_option(Some(Duration::from_nanos(1))).as_deref(), + Some("-c statement_timeout=1ms") + ); assert_eq!( timeout_option(Some(Duration::from_micros(500))).as_deref(), Some("-c statement_timeout=1ms")