From 71c8dd47f048edf2b498e5c547f5cb9b418db1e1 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 9 Jul 2026 08:35:36 -0400 Subject: [PATCH 1/5] updates --- crates/client-api/src/routes/mod.rs | 50 ++++++++++++++++++++++++++- crates/core/src/sql/execute.rs | 33 ++++++++++++++++++ crates/core/src/worker_metrics/mod.rs | 20 +++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/crates/client-api/src/routes/mod.rs b/crates/client-api/src/routes/mod.rs index 08e1e73cb77..675b64e16d2 100644 --- a/crates/client-api/src/routes/mod.rs +++ b/crates/client-api/src/routes/mod.rs @@ -1,4 +1,9 @@ +use axum::body::HttpBody as _; +use axum::extract::{MatchedPath, Request}; +use axum::middleware::Next; +use axum::response::Response; use http::header; +use spacetimedb::worker_metrics::WORKER_METRICS; use tower_http::cors; use crate::{Authorization, ControlStateDelegate, NodeDelegate}; @@ -18,6 +23,44 @@ use self::{database::DatabaseRoutes, identity::IdentityRoutes}; /// establish a connection to SpacetimeDB. This API call doesn't actually do anything. pub async fn ping(_auth: crate::auth::SpacetimeAuthHeader) {} +/// Records request count, latency and body sizes per HTTP route. +async fn http_metrics_middleware(req: Request, next: Next) -> Response { + let route = req + .extensions() + .get::() + .map_or_else(|| "".to_owned(), |p| p.as_str().to_owned()); + let method = req.method().as_str().to_owned(); + + let request_body_bytes = req + .headers() + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + + let start = std::time::Instant::now(); + let res = next.run(req).await; + + WORKER_METRICS + .http_requests + .with_label_values(&route, &method, res.status().as_str()) + .inc(); + WORKER_METRICS + .http_request_duration + .with_label_values(&route) + .observe(start.elapsed().as_secs_f64()); + if let Some(n) = request_body_bytes { + WORKER_METRICS.http_request_body_bytes.with_label_values(&route).inc_by(n); + } + if let Some(n) = res.body().size_hint().exact() { + WORKER_METRICS + .http_response_body_bytes + .with_label_values(&route) + .inc_by(n); + } + + res +} + #[allow(clippy::let_and_return)] pub fn router( ctx: &S, @@ -44,6 +87,11 @@ where .allow_origin(cors::Any); axum::Router::new() - .nest("/v1", router.layer(cors)) + .nest( + "/v1", + router + .layer(axum::middleware::from_fn(http_metrics_middleware)) + .layer(cors), + ) .nest("/internal", internal::router()) } diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 7fe2fe229f8..4c2c9bdb44e 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -23,6 +23,7 @@ use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_lib::Timestamp; use spacetimedb_lib::{AlgebraicType, ProductType, ProductValue}; use spacetimedb_query::{compile_sql_stmt, execute_dml_stmt, execute_select_stmt}; +use spacetimedb_sats::bsatn; use spacetimedb_sats::raw_identifier::RawIdentifier; use tokio::sync::oneshot; @@ -129,6 +130,9 @@ fn run_inner( Ok(plan) })?; + // Charge egress, using BSATN size for parity with WebSocket queries. + metrics.bytes_sent_to_clients += rows.iter().map(|row| bsatn::to_len(row).unwrap_or(0)).sum::(); + // Update transaction metrics tx.metrics.merge(metrics); @@ -1686,4 +1690,33 @@ pub(crate) mod tests { Ok(()) } + + // Selects should charge egress for the rows they return + #[test] + fn test_select_charges_egress() -> ResultTest<()> { + let db = TestDB::durable()?; + + let table_id = db.create_table_for_test("T", &[("a", AlgebraicType::U8)], &[])?; + with_auto_commit(&db, |tx| -> Result<_, DBError> { + for i in 0..4u8 { + insert(&db, tx, table_id, &product!(i))?; + } + Ok(()) + })?; + + let rt = db.runtime().expect("runtime should be there"); + + let server = Identity::from_claims("issuer", "server"); + let auth = AuthCtx::new(server, server); + + let mut head = Vec::new(); + let result = rt.block_on(run(db.clone(), "SELECT * FROM T".to_string(), auth, None, None, &mut head))?; + + assert_eq!(result.rows.len(), 4); + let expected: usize = result.rows.iter().map(|row| bsatn::to_len(row).unwrap()).sum(); + assert!(expected > 0); + assert_eq!(result.metrics.bytes_sent_to_clients, expected); + + Ok(()) + } } diff --git a/crates/core/src/worker_metrics/mod.rs b/crates/core/src/worker_metrics/mod.rs index e99bd196b63..8c265665999 100644 --- a/crates/core/src/worker_metrics/mod.rs +++ b/crates/core/src/worker_metrics/mod.rs @@ -51,6 +51,26 @@ metrics_group!( #[labels(database_identity: Identity, protocol: str)] pub websocket_request_msg_size: HistogramVec, + #[name = spacetime_http_requests_total] + #[help = "The cumulative number of HTTP requests, by matched route, method and response status"] + #[labels(route: str, method: str, status: str)] + pub http_requests: IntCounterVec, + + #[name = spacetime_http_request_duration_sec] + #[help = "The time (in seconds) spent handling an HTTP request, by matched route"] + #[labels(route: str)] + pub http_request_duration: HistogramVec, + + #[name = spacetime_http_request_body_bytes_total] + #[help = "The cumulative number of HTTP request body bytes received, by matched route"] + #[labels(route: str)] + pub http_request_body_bytes: IntCounterVec, + + #[name = spacetime_http_response_body_bytes_total] + #[help = "The cumulative number of HTTP response body bytes sent, by matched route"] + #[labels(route: str)] + pub http_response_body_bytes: IntCounterVec, + #[name = jemalloc_active_bytes] #[help = "Number of bytes in jemallocs heap"] #[labels(node_id: str)] From 10e942b92644d063afc37d926d19960d00efd02e Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:17:00 -0400 Subject: [PATCH 2/5] test: match existing convention for throwaway head arg --- crates/core/src/sql/execute.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 4c2c9bdb44e..25fc73507f2 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1709,8 +1709,7 @@ pub(crate) mod tests { let server = Identity::from_claims("issuer", "server"); let auth = AuthCtx::new(server, server); - let mut head = Vec::new(); - let result = rt.block_on(run(db.clone(), "SELECT * FROM T".to_string(), auth, None, None, &mut head))?; + let result = rt.block_on(run(db.clone(), "SELECT * FROM T".to_string(), auth, None, None, &mut vec![]))?; assert_eq!(result.rows.len(), 4); let expected: usize = result.rows.iter().map(|row| bsatn::to_len(row).unwrap()).sum(); From 6864080cecdf681e61a6a626d4e741e2f3787a80 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:14:44 -0400 Subject: [PATCH 3/5] http body + bucketing --- Cargo.lock | 1 + Cargo.toml | 1 + crates/client-api/Cargo.toml | 1 + crates/client-api/src/routes/mod.rs | 95 +++++++++++++++++++++-------- 4 files changed, 74 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40ddf5ea619..f9361218716 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8118,6 +8118,7 @@ dependencies = [ "futures", "headers", "http", + "http-body", "http-body-util", "humantime", "hyper", diff --git a/Cargo.toml b/Cargo.toml index 842332340d8..b4b3f6286d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -223,6 +223,7 @@ hex = "0.4.3" home = "0.5" hostname = "^0.3" http = "1.0" +http-body = "1.0" http-body-util= "0.1.3" humantime = "2.3" hyper = "1.0" diff --git a/crates/client-api/Cargo.toml b/crates/client-api/Cargo.toml index afbb5a2a340..3cd1f3fb466 100644 --- a/crates/client-api/Cargo.toml +++ b/crates/client-api/Cargo.toml @@ -16,6 +16,7 @@ spacetimedb-paths.workspace = true spacetimedb-schema.workspace = true base64.workspace = true +http-body.workspace = true http-body-util.workspace = true tokio = { version = "1.2", features = ["full"] } lazy_static = "1.4.0" diff --git a/crates/client-api/src/routes/mod.rs b/crates/client-api/src/routes/mod.rs index 675b64e16d2..3689fbbb876 100644 --- a/crates/client-api/src/routes/mod.rs +++ b/crates/client-api/src/routes/mod.rs @@ -1,8 +1,13 @@ -use axum::body::HttpBody as _; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use axum::body::{Body, Bytes, HttpBody}; use axum::extract::{MatchedPath, Request}; use axum::middleware::Next; use axum::response::Response; use http::header; +use http_body::Frame; +use ::prometheus::IntCounter; use spacetimedb::worker_metrics::WORKER_METRICS; use tower_http::cors; @@ -23,42 +28,84 @@ use self::{database::DatabaseRoutes, identity::IdentityRoutes}; /// establish a connection to SpacetimeDB. This API call doesn't actually do anything. pub async fn ping(_auth: crate::auth::SpacetimeAuthHeader) {} +/// A body wrapper that counts bytes as they are actually transferred. +struct CountingBody { + inner: Body, + counter: IntCounter, +} + +impl HttpBody for CountingBody { + type Data = Bytes; + type Error = axum::Error; + + fn poll_frame(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll, Self::Error>>> { + let poll = Pin::new(&mut self.inner).poll_frame(cx); + if let Poll::Ready(Some(Ok(frame))) = &poll { + if let Some(data) = frame.data_ref() { + self.counter.inc_by(data.len() as u64); + } + } + poll + } + + fn size_hint(&self) -> http_body::SizeHint { + self.inner.size_hint() + } + + fn is_end_stream(&self) -> bool { + self.inner.is_end_stream() + } +} + +impl CountingBody { + fn wrap(counter: IntCounter) -> impl FnOnce(Body) -> Body { + move |inner| Body::new(CountingBody { inner, counter }) + } +} + +/// Returns the method as a static label value, +/// bucketing non-standard methods to keep label cardinality bounded. +fn method_label(method: &http::Method) -> &'static str { + match method.as_str() { + "GET" => "GET", + "POST" => "POST", + "PUT" => "PUT", + "DELETE" => "DELETE", + "HEAD" => "HEAD", + "OPTIONS" => "OPTIONS", + "PATCH" => "PATCH", + "CONNECT" => "CONNECT", + "TRACE" => "TRACE", + _ => "OTHER", + } +} + /// Records request count, latency and body sizes per HTTP route. async fn http_metrics_middleware(req: Request, next: Next) -> Response { - let route = req - .extensions() - .get::() - .map_or_else(|| "".to_owned(), |p| p.as_str().to_owned()); - let method = req.method().as_str().to_owned(); - - let request_body_bytes = req - .headers() - .get(header::CONTENT_LENGTH) - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()); + let Some(route) = req.extensions().get::().cloned() else { + return next.run(req).await; + }; + + let method = method_label(req.method()); + + let request_body_bytes = WORKER_METRICS.http_request_body_bytes.with_label_values(route.as_str()); + let response_body_bytes = WORKER_METRICS.http_response_body_bytes.with_label_values(route.as_str()); + + let req = req.map(CountingBody::wrap(request_body_bytes)); let start = std::time::Instant::now(); let res = next.run(req).await; WORKER_METRICS .http_requests - .with_label_values(&route, &method, res.status().as_str()) + .with_label_values(route.as_str(), method, res.status().as_str()) .inc(); WORKER_METRICS .http_request_duration - .with_label_values(&route) + .with_label_values(route.as_str()) .observe(start.elapsed().as_secs_f64()); - if let Some(n) = request_body_bytes { - WORKER_METRICS.http_request_body_bytes.with_label_values(&route).inc_by(n); - } - if let Some(n) = res.body().size_hint().exact() { - WORKER_METRICS - .http_response_body_bytes - .with_label_values(&route) - .inc_by(n); - } - res + res.map(CountingBody::wrap(response_body_bytes)) } #[allow(clippy::let_and_return)] From 2ac04e3d2baa8dfb8ef0f1e4fdd7bbee0304e405 Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:25:19 -0400 Subject: [PATCH 4/5] lints --- crates/client-api/src/routes/mod.rs | 6 ++++-- crates/core/src/sql/execute.rs | 9 ++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/client-api/src/routes/mod.rs b/crates/client-api/src/routes/mod.rs index 3689fbbb876..d922f95ce18 100644 --- a/crates/client-api/src/routes/mod.rs +++ b/crates/client-api/src/routes/mod.rs @@ -1,13 +1,13 @@ use std::pin::Pin; use std::task::{Context, Poll}; +use ::prometheus::IntCounter; use axum::body::{Body, Bytes, HttpBody}; use axum::extract::{MatchedPath, Request}; use axum::middleware::Next; use axum::response::Response; use http::header; use http_body::Frame; -use ::prometheus::IntCounter; use spacetimedb::worker_metrics::WORKER_METRICS; use tower_http::cors; @@ -89,7 +89,9 @@ async fn http_metrics_middleware(req: Request, next: Next) -> Response { let method = method_label(req.method()); let request_body_bytes = WORKER_METRICS.http_request_body_bytes.with_label_values(route.as_str()); - let response_body_bytes = WORKER_METRICS.http_response_body_bytes.with_label_values(route.as_str()); + let response_body_bytes = WORKER_METRICS + .http_response_body_bytes + .with_label_values(route.as_str()); let req = req.map(CountingBody::wrap(request_body_bytes)); diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 25fc73507f2..ee24ce0024d 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -1709,7 +1709,14 @@ pub(crate) mod tests { let server = Identity::from_claims("issuer", "server"); let auth = AuthCtx::new(server, server); - let result = rt.block_on(run(db.clone(), "SELECT * FROM T".to_string(), auth, None, None, &mut vec![]))?; + let result = rt.block_on(run( + db.clone(), + "SELECT * FROM T".to_string(), + auth, + None, + None, + &mut vec![], + ))?; assert_eq!(result.rows.len(), 4); let expected: usize = result.rows.iter().map(|row| bsatn::to_len(row).unwrap()).sum(); From 50e110fae9f169b57cd0771ba5b17037233c786b Mon Sep 17 00:00:00 2001 From: bradleyshep <148254416+bradleyshep@users.noreply.github.com> Date: Thu, 9 Jul 2026 10:26:37 -0400 Subject: [PATCH 5/5] Update mod.rs --- crates/client-api/src/routes/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/client-api/src/routes/mod.rs b/crates/client-api/src/routes/mod.rs index d922f95ce18..3a1382e0a40 100644 --- a/crates/client-api/src/routes/mod.rs +++ b/crates/client-api/src/routes/mod.rs @@ -40,10 +40,10 @@ impl HttpBody for CountingBody { fn poll_frame(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll, Self::Error>>> { let poll = Pin::new(&mut self.inner).poll_frame(cx); - if let Poll::Ready(Some(Ok(frame))) = &poll { - if let Some(data) = frame.data_ref() { - self.counter.inc_by(data.len() as u64); - } + if let Poll::Ready(Some(Ok(frame))) = &poll + && let Some(data) = frame.data_ref() + { + self.counter.inc_by(data.len() as u64); } poll }