From 4f7a791e62eed58dd277a06326f77f68913ff51e Mon Sep 17 00:00:00 2001 From: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Date: Fri, 31 Jul 2026 11:16:29 -0700 Subject: [PATCH 01/51] feat: add durable community deletion worker Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> --- Cargo.lock | 22 + Cargo.toml | 2 + crates/buzz-admin/Cargo.toml | 1 + crates/buzz-admin/src/deletions.rs | 19 + crates/buzz-admin/src/main.rs | 8 + crates/buzz-db/src/deletion.rs | 1910 +++++++++++++++++ crates/buzz-db/src/lib.rs | 13 +- crates/buzz-db/src/migration.rs | 75 +- crates/buzz-deletion/Cargo.toml | 23 + crates/buzz-deletion/src/lib.rs | 752 +++++++ crates/buzz-media/src/bucket_index.rs | 147 ++ crates/buzz-media/src/lib.rs | 8 +- crates/buzz-media/src/storage.rs | 88 + crates/buzz-relay/Cargo.toml | 1 + crates/buzz-relay/src/api/git/transport.rs | 18 + crates/buzz-relay/src/api/media.rs | 5 + .../src/handlers/command_executor.rs | 6 + crates/buzz-relay/src/handlers/event.rs | 28 + crates/buzz-relay/src/handlers/ingest.rs | 7 + crates/buzz-relay/src/main.rs | 1 + crates/buzz-relay/src/mesh_boot.rs | 18 +- crates/buzz-relay/src/push_runtime.rs | 14 + crates/buzz-relay/src/tunnel/directory.rs | 36 +- crates/buzz-relay/src/workflow_sink.rs | 5 + crates/buzz-workflow/Cargo.toml | 1 + crates/buzz-workflow/src/executor.rs | 13 + deploy/charts/buzz/README.md | 31 + .../templates/deletion-worker-deployment.yaml | 95 + .../buzz/tests/deletion_worker_test.yaml | 45 + deploy/charts/buzz/tests/render_test.yaml | 1 + deploy/charts/buzz/values.yaml | 23 + migrations/0028_community_deletion.sql | 239 +++ schema/schema.sql | 176 ++ 33 files changed, 3821 insertions(+), 10 deletions(-) create mode 100644 crates/buzz-admin/src/deletions.rs create mode 100644 crates/buzz-db/src/deletion.rs create mode 100644 crates/buzz-deletion/Cargo.toml create mode 100644 crates/buzz-deletion/src/lib.rs create mode 100644 deploy/charts/buzz/templates/deletion-worker-deployment.yaml create mode 100644 deploy/charts/buzz/tests/deletion_worker_test.yaml create mode 100644 migrations/0028_community_deletion.sql diff --git a/Cargo.lock b/Cargo.lock index 73ecb249d4..1420c13305 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -838,6 +838,7 @@ dependencies = [ "buzz-auth", "buzz-core", "buzz-db", + "buzz-deletion", "buzz-media", "buzz-pubsub", "buzz-search", @@ -1019,6 +1020,25 @@ dependencies = [ "uuid", ] +[[package]] +name = "buzz-deletion" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "buzz-core", + "buzz-db", + "buzz-media", + "clap", + "deadpool-redis", + "redis", + "serde", + "serde_json", + "tokio", + "tokio-util", + "uuid", +] + [[package]] name = "buzz-dev-mcp" version = "0.1.0" @@ -1191,6 +1211,7 @@ dependencies = [ "buzz-conformance", "buzz-core", "buzz-db", + "buzz-deletion", "buzz-media", "buzz-pubsub", "buzz-relay-mesh", @@ -1348,6 +1369,7 @@ version = "0.1.0" dependencies = [ "buzz-core", "buzz-db", + "buzz-deletion", "chrono", "cron", "dashmap", diff --git a/Cargo.toml b/Cargo.toml index cc1dd0f9df..f22330c683 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "crates/buzz-test-client", "crates/buzz-ws-client", "crates/buzz-admin", + "crates/buzz-deletion", "crates/buzz-workflow", "crates/buzz-media", "crates/buzz-cli", @@ -134,6 +135,7 @@ schemars = { version = "1", default-features = false } buzz-core = { path = "crates/buzz-core" } buzz-conformance = { path = "crates/buzz-conformance" } buzz-db = { path = "crates/buzz-db" } +buzz-deletion = { path = "crates/buzz-deletion" } buzz-auth = { path = "crates/buzz-auth" } buzz-pubsub = { path = "crates/buzz-pubsub" } buzz-search = { path = "crates/buzz-search" } diff --git a/crates/buzz-admin/Cargo.toml b/crates/buzz-admin/Cargo.toml index 7a69e146bb..00c2804cbc 100644 --- a/crates/buzz-admin/Cargo.toml +++ b/crates/buzz-admin/Cargo.toml @@ -13,6 +13,7 @@ path = "src/main.rs" [dependencies] buzz-db = { workspace = true } +buzz-deletion = { workspace = true } buzz-core = { workspace = true } buzz-auth = { workspace = true } buzz-pubsub = { workspace = true } diff --git a/crates/buzz-admin/src/deletions.rs b/crates/buzz-admin/src/deletions.rs new file mode 100644 index 0000000000..95d07588a2 --- /dev/null +++ b/crates/buzz-admin/src/deletions.rs @@ -0,0 +1,19 @@ +//! Thin `buzz-admin deletions` adapter. + +pub use buzz_deletion::Command as DeletionsCommand; + +/// Delegate to the shared durable deletion engine. +pub async fn run(command: DeletionsCommand) -> anyhow::Result { + buzz_deletion::run(command).await +} + +#[cfg(test)] +mod tests { + use clap::Parser; + + #[test] + fn default_command_does_not_start_a_deletion_worker() { + let command = crate::Cli::try_parse_from(["buzz-admin", "list-members"]); + assert!(command.is_ok()); + } +} diff --git a/crates/buzz-admin/src/main.rs b/crates/buzz-admin/src/main.rs index bb30ddfae4..580d586591 100644 --- a/crates/buzz-admin/src/main.rs +++ b/crates/buzz-admin/src/main.rs @@ -20,6 +20,8 @@ //! newest timestamp and collide on the bumped second. run.sh serialization is //! the guard against parallel adds (e.g. `xargs -P`). +mod deletions; + use std::sync::Arc; use anyhow::Result; @@ -81,6 +83,11 @@ enum Command { #[command(subcommand)] command: ProductFeedbackCommand, }, + /// Durable CLI-only whole-community deletion control plane. + Deletions { + #[command(subcommand)] + command: deletions::DeletionsCommand, + }, /// Emit kind:39000/39002 events for channels missing them. /// /// Channels created via direct SQL (seed scripts, pre-migration data) won't @@ -148,6 +155,7 @@ async fn run(cli: Cli) -> Result { Command::ProductFeedback { command: ProductFeedbackCommand::List { limit }, } => cmd_list_product_feedback(limit).await, + Command::Deletions { command } => deletions::run(command).await, Command::ReconcileChannels { relay_key } => { reconcile_channels(relay_key).await?; Ok(0) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs new file mode 100644 index 0000000000..a8edb3a169 --- /dev/null +++ b/crates/buzz-db/src/deletion.rs @@ -0,0 +1,1910 @@ +//! Durable whole-community deletion lifecycle and PostgreSQL adapter. +//! +//! This module owns request inventory, approval, claims, fencing, checkpoints, +//! retries, tombstoning, and logical verification. CLI claim-loop policy and +//! external storage adapters live above it; they never implement state changes. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::str::FromStr; +use std::time::Duration; + +use buzz_core::CommunityId; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use sqlx::{AssertSqlSafe, PgPool, Postgres, Row, Transaction}; +use uuid::Uuid; + +use crate::error::{DbError, Result}; + +/// Exact desired catalog revision understood by this deletion engine. +pub const CATALOG_REVISION: i32 = 1; +/// Highest SQL migration version whose tenant catalog this engine understands. +pub const EXPECTED_MIGRATION_VERSION: i64 = 27; +/// Default PostgreSQL lease duration for one claimed deletion request. +pub const DEFAULT_LEASE_DURATION: Duration = Duration::from_secs(60); +/// Durable name of the schema manifest's PostgreSQL component. +pub const POSTGRES_STORE_NAME: &str = "postgres"; +/// Durable name of the object-store manifest component. +pub const OBJECT_STORE_NAME: &str = "object_store"; +/// Durable name of the Redis/cache manifest component. +pub const REDIS_STORE_NAME: &str = "redis"; + +/// Control-plane tables that survive the community data purge. +pub const CONTROL_PLANE_TABLES: &[&str] = &[ + "community_deletion_approvals", + "community_deletion_checkpoints", + "community_deletion_executor_heartbeats", + "community_deletion_requests", + "community_deletion_retention_exceptions", +]; + +/// Expected community-scoped tables purged by V1. +/// +/// Catalog inventory compares the live database against this exact set before +/// approval and again before PostgreSQL purge. A new tenant table therefore +/// blocks deletion until this manifest is intentionally updated. +pub const EXPECTED_SCOPED_TABLES: &[&str] = &[ + "api_tokens", + "archived_identities", + "audit_log", + "channel_members", + "channels", + "community_bans", + "delivery_log", + "event_mentions", + "events", + "git_repo_names", + "join_policy_acceptances", + "moderation_actions", + "moderation_reports", + "parameterized_event_watermarks", + "product_feedback", + "rate_limit_violations", + "pubkey_allowlist", + "push_leases", + "push_match_queue", + "push_wake_outbox", + "reactions", + "relay_invites", + "relay_members", + "scheduled_workflow_fires", + "subscriptions", + "thread_metadata", + "users", + "workflow_approvals", + "workflow_runs", + "workflows", +]; + +/// Foreign-key-safe child-before-parent order for the PostgreSQL purge. +pub const PURGE_SCOPED_TABLES: &[&str] = &[ + "workflow_approvals", + "scheduled_workflow_fires", + "workflow_runs", + "push_wake_outbox", + "join_policy_acceptances", + "moderation_reports", + "subscriptions", + "api_tokens", + "channel_members", + "thread_metadata", + "moderation_actions", + "workflows", + "event_mentions", + "reactions", + "push_match_queue", + "push_leases", + "relay_invites", + "product_feedback", + "rate_limit_violations", + "delivery_log", + "events", + "parameterized_event_watermarks", + "git_repo_names", + "archived_identities", + "audit_log", + "community_bans", + "pubkey_allowlist", + "relay_members", + "users", + "channels", +]; + +/// Fixed lifecycle order. There are no backwards or skipping transitions. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum DeletionStage { + /// Request exists but has not frozen its inventory. + Submitted, + /// PostgreSQL and storage inventory has been frozen. + Inventoried, + /// An operator explicitly approved the frozen inventory digest. + Approved, + /// Universal serving-path write fence is active. + Fenced, + /// In-flight serving writes have drained behind the durable fence. + Drained, + /// Tenant-owned S3/media and Git pointer bindings were removed. + BindingsRemoved, + /// Tenant-scoped PostgreSQL rows were purged. + PostgresPurged, + /// Redis/community process-cache namespace was purged. + CachePurged, + /// Cross-store logical absence was verified. + LogicallyVerified, + /// Logical deletion complete; shared CAS physical expiry is deferred. + RetentionPending, +} + +impl DeletionStage { + /// Next legal stage, if this is not terminal. + pub const fn next(self) -> Option { + match self { + Self::Submitted => Some(Self::Inventoried), + Self::Inventoried => Some(Self::Approved), + Self::Approved => Some(Self::Fenced), + Self::Fenced => Some(Self::Drained), + Self::Drained => Some(Self::BindingsRemoved), + Self::BindingsRemoved => Some(Self::PostgresPurged), + Self::PostgresPurged => Some(Self::CachePurged), + Self::CachePurged => Some(Self::LogicallyVerified), + Self::LogicallyVerified => Some(Self::RetentionPending), + Self::RetentionPending => None, + } + } + + /// Whether execution may claim this stage. + pub const fn runnable(self) -> bool { + matches!( + self, + Self::Approved + | Self::Fenced + | Self::Drained + | Self::BindingsRemoved + | Self::PostgresPurged + | Self::CachePurged + | Self::LogicallyVerified + ) + } +} + +impl fmt::Display for DeletionStage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let value = match self { + Self::Submitted => "submitted", + Self::Inventoried => "inventoried", + Self::Approved => "approved", + Self::Fenced => "fenced", + Self::Drained => "drained", + Self::BindingsRemoved => "bindings_removed", + Self::PostgresPurged => "postgres_purged", + Self::CachePurged => "cache_purged", + Self::LogicallyVerified => "logically_verified", + Self::RetentionPending => "retention_pending", + }; + f.write_str(value) + } +} + +impl FromStr for DeletionStage { + type Err = DbError; + + fn from_str(value: &str) -> std::result::Result { + match value { + "submitted" => Ok(Self::Submitted), + "inventoried" => Ok(Self::Inventoried), + "approved" => Ok(Self::Approved), + "fenced" => Ok(Self::Fenced), + "drained" => Ok(Self::Drained), + "bindings_removed" => Ok(Self::BindingsRemoved), + "postgres_purged" => Ok(Self::PostgresPurged), + "cache_purged" => Ok(Self::CachePurged), + "logically_verified" => Ok(Self::LogicallyVerified), + "retention_pending" => Ok(Self::RetentionPending), + other => Err(DbError::InvalidData(format!( + "unknown community deletion stage: {other}" + ))), + } + } +} + +/// Durable community deletion request. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionRequest { + /// Request identifier. + pub id: Uuid, + /// Target community. + #[serde(serialize_with = "serialize_community_id")] + pub community_id: CommunityId, + /// Permanently reserved canonical host. + pub community_host: String, + /// Current lifecycle stage. + pub stage: DeletionStage, + /// Operator identity that submitted the request. + pub requested_by: String, + /// Optional request reason. + pub reason: Option, + /// Frozen catalog manifest. + pub schema_manifest: Option, + /// Frozen storage manifest. + pub storage_manifest: Option, + /// Frozen inventory aggregate. + pub inventory_manifest: Option, + /// Hex SHA-256 of the frozen inventory. + pub inventory_digest: Option, + /// Durable community fence generation. + pub fence_generation: Option, + /// Current claim owner. + pub lease_owner: Option, + /// Monotonic claim generation. + pub lease_generation: i64, + /// Claim expiry. + pub lease_until: Option>, + /// Number of claims. + pub attempts: i32, + /// Number of failed execution units. + pub retry_count: i32, + /// Last bounded error. + pub last_error: Option, + /// Permanent fail-closed block reason. + pub blocked_reason: Option, + /// Submission time. + pub created_at: DateTime, + /// Last lifecycle update. + pub updated_at: DateTime, + /// Terminal logical-deletion time. + pub completed_at: Option>, +} + +/// Frozen PostgreSQL catalog inventory. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SchemaManifest { + /// Engine catalog revision. + pub revision: i32, + /// Live SQLx migration version. + pub migration_version: i64, + /// Sorted community-scoped table names. + pub scoped_tables: Vec, + /// Per-table row counts for the target. + pub row_counts: BTreeMap, + /// Sorted tables with the universal write-fence trigger. + pub fenced_tables: Vec, +} + +/// Frozen storage inventory supplied by the object-store adapter. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StorageManifest { + /// Adapter schema version. + pub version: i32, + /// Every tenant-owned key that will be removed, sorted. + pub tenant_keys: Vec, + /// Git pointer keys among tenant keys. + pub git_pointer_keys: Vec, + /// Media sidecar keys among tenant keys. + pub media_sidecar_keys: Vec, + /// Media upload-record keys among tenant keys. + pub media_upload_keys: Vec, + /// Reserved for future retention exceptions. Empty in V1: shared CAS is + /// fleet-wide and intentionally outside per-community inventory. + pub retained_shared_cas_keys: Vec, + /// Fleet keys unknown to the current taxonomy. Non-empty fails inventory. + pub unknown_keys: Vec, + /// Keys whose current object version cannot be safely removed. Non-empty fails. + pub unsupported_version_keys: Vec, +} + +/// Full frozen inventory approved at the destructive boundary. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FrozenInventory { + /// PostgreSQL catalog state. + pub schema: SchemaManifest, + /// Object-store state. + pub storage: StorageManifest, +} + +impl FrozenInventory { + /// Canonical JSON bytes and SHA-256 digest used to bind approval. + pub fn digest(&self) -> Result> { + Ok(Sha256::digest(serde_json::to_vec(self)?).to_vec()) + } +} + +/// One durable unit checkpoint. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionCheckpoint { + /// Stage containing the unit. + pub stage: String, + /// Stable unit key. + pub unit_key: String, + /// `started`, `completed`, or `failed`. + pub status: String, + /// Claim generation that last touched it. + pub lease_generation: i64, + /// Attempt count for this unit. + pub attempts: i32, + /// Structured bounded details. + pub detail: serde_json::Value, + /// Last failure. + pub error: Option, + /// Start time. + pub started_at: DateTime, + /// Completion time. + pub completed_at: Option>, +} + +/// Full inspect response. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionInspection { + /// Durable request. + pub request: DeletionRequest, + /// Explicit approval evidence, if present. + pub approval: Option, + /// Unit checkpoints. + pub checkpoints: Vec, + /// Retention exceptions/holds. + pub retention_exceptions: Vec, +} + +/// Explicit approval evidence. +#[derive(Debug, Clone, Serialize)] +pub struct DeletionApproval { + /// Hex frozen inventory digest. + pub inventory_digest: String, + /// Approving operator identity. + pub approved_by: String, + /// Optional approval note. + pub note: Option, + /// Approval timestamp. + pub approved_at: DateTime, +} + +/// Physical-expiry retention exception. +#[derive(Debug, Clone, Serialize)] +pub struct RetentionException { + /// Stable hold/exception key. + pub exception_key: String, + /// Reason it survives logical deletion. + pub reason: String, + /// Optional expiry. + pub expires_at: Option>, + /// Creating operator. + pub created_by: String, + /// Creation timestamp. + pub created_at: DateTime, +} + +/// Monotonic lease token required by every execution mutation. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LeaseToken { + /// Request id. + pub request_id: Uuid, + /// Executor identity. + pub owner: String, + /// Monotonic lease generation. + pub generation: i64, + /// Target community. + pub community_id: CommunityId, + /// Community fence generation, once fenced. + pub fence_generation: Option, +} + +/// A claimed request with its durable token. +#[derive(Debug, Clone)] +pub struct ClaimedDeletion { + /// Request snapshot. + pub request: DeletionRequest, + /// Required token. + pub lease: LeaseToken, +} + +/// Held shared advisory lock proving a serving write began before fencing. +/// +/// Dropping the guard rolls back its read-only transaction and releases the +/// lock. The destructive fence takes the matching exclusive lock, so it cannot +/// advance while an S3/Redis/webhook side effect still holds this guard. +pub struct ServingWriteGuard { + _tx: Transaction<'static, Postgres>, +} + +/// PostgreSQL deletion adapter. Clone is cheap. +#[derive(Clone)] +pub struct DeletionStore { + pool: PgPool, +} + +impl DeletionStore { + /// Construct from the writer pool used by [`crate::Db`]. + pub(crate) fn new(pool: PgPool) -> Self { + Self { pool } + } + + /// Persist a request. Only active non-tombstone communities may be submitted. + pub async fn submit( + &self, + community_host: &str, + requested_by: &str, + reason: Option<&str>, + ) -> Result { + let row = sqlx::query( + r#" + WITH target AS ( + SELECT id, host + FROM communities + WHERE lower(host) = lower($1) + AND deletion_state = 'active' + AND deleted_at IS NULL + ), inserted AS ( + INSERT INTO community_deletion_requests + (community_id, community_host, requested_by, reason) + SELECT id, host, $2, $3 FROM target + ON CONFLICT (community_id) DO NOTHING + RETURNING * + ) + SELECT * FROM inserted + UNION ALL + SELECT request.* + FROM community_deletion_requests request + JOIN target ON target.id = request.community_id + WHERE request.stage = 'submitted' + AND request.requested_by = $2 + AND NOT EXISTS (SELECT 1 FROM inserted) + LIMIT 1 + "#, + ) + .bind(community_host) + .bind(requested_by) + .bind(reason) + .fetch_optional(&self.pool) + .await?; + match row { + Some(row) => row_to_request(row), + None => Err(DbError::InvalidData(format!( + "community {community_host:?} is missing, already requested, fenced, or tombstoned" + ))), + } + } + + /// List requests newest first with a hard bound. + pub async fn list(&self, limit: i64) -> Result> { + let rows = sqlx::query( + "SELECT * FROM community_deletion_requests ORDER BY created_at DESC LIMIT $1", + ) + .bind(limit.clamp(1, 1000)) + .fetch_all(&self.pool) + .await?; + rows.into_iter().map(row_to_request).collect() + } + + /// Read one request. + pub async fn get(&self, request_id: Uuid) -> Result { + let row = sqlx::query("SELECT * FROM community_deletion_requests WHERE id = $1") + .bind(request_id) + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| DbError::NotFound(format!("community deletion {request_id}")))?; + row_to_request(row) + } + + /// Inspect request, approval, checkpoints, and retention holds. + pub async fn inspect(&self, request_id: Uuid) -> Result { + let request = self.get(request_id).await?; + let approval_row = sqlx::query( + "SELECT inventory_digest, approved_by, note, approved_at \ + FROM community_deletion_approvals WHERE request_id = $1", + ) + .bind(request_id) + .fetch_optional(&self.pool) + .await?; + let approval = approval_row + .map(|row| { + Ok::(DeletionApproval { + inventory_digest: hex::encode(row.try_get::, _>("inventory_digest")?), + approved_by: row.try_get("approved_by")?, + note: row.try_get("note")?, + approved_at: row.try_get("approved_at")?, + }) + }) + .transpose()?; + let checkpoints = sqlx::query( + "SELECT stage, unit_key, status, lease_generation, attempts, detail, error, \ + started_at, completed_at \ + FROM community_deletion_checkpoints WHERE request_id = $1 ORDER BY sequence", + ) + .bind(request_id) + .fetch_all(&self.pool) + .await? + .into_iter() + .map(|row| { + Ok(DeletionCheckpoint { + stage: row.try_get("stage")?, + unit_key: row.try_get("unit_key")?, + status: row.try_get("status")?, + lease_generation: row.try_get("lease_generation")?, + attempts: row.try_get("attempts")?, + detail: row.try_get("detail")?, + error: row.try_get("error")?, + started_at: row.try_get("started_at")?, + completed_at: row.try_get("completed_at")?, + }) + }) + .collect::>>()?; + let retention_exceptions = sqlx::query( + "SELECT exception_key, reason, expires_at, created_by, created_at \ + FROM community_deletion_retention_exceptions WHERE request_id = $1 \ + ORDER BY exception_key", + ) + .bind(request_id) + .fetch_all(&self.pool) + .await? + .into_iter() + .map(|row| { + Ok(RetentionException { + exception_key: row.try_get("exception_key")?, + reason: row.try_get("reason")?, + expires_at: row.try_get("expires_at")?, + created_by: row.try_get("created_by")?, + created_at: row.try_get("created_at")?, + }) + }) + .collect::>>()?; + Ok(DeletionInspection { + request, + approval, + checkpoints, + retention_exceptions, + }) + } + + /// Build and validate a live PostgreSQL schema inventory. + pub async fn inventory_schema(&self, community: CommunityId) -> Result { + let migration_version: i64 = sqlx::query_scalar( + "SELECT COALESCE(max(version), 0) FROM _sqlx_migrations WHERE success", + ) + .fetch_one(&self.pool) + .await?; + if migration_version != EXPECTED_MIGRATION_VERSION { + return Err(DbError::InvalidData(format!( + "community deletion schema migration drift: expected {EXPECTED_MIGRATION_VERSION}, got {migration_version}" + ))); + } + + let live_tables = self.live_scoped_tables().await?; + let expected = EXPECTED_SCOPED_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + if live_tables != expected { + let missing = expected + .difference(&live_tables) + .cloned() + .collect::>(); + let unknown = live_tables + .difference(&expected) + .cloned() + .collect::>(); + return Err(DbError::InvalidData(format!( + "community deletion catalog drift (missing={}, unknown={})", + missing.join(","), + unknown.join(",") + ))); + } + + let fenced_tables = self.live_fenced_tables().await?; + if fenced_tables != expected { + let missing = expected + .difference(&fenced_tables) + .cloned() + .collect::>(); + return Err(DbError::InvalidData(format!( + "community deletion write-fence drift (missing={})", + missing.join(",") + ))); + } + + let mut row_counts = BTreeMap::new(); + for table in &live_tables { + let sql = format!("SELECT count(*) FROM {table} WHERE community_id = $1"); + let count: i64 = sqlx::query_scalar(AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await?; + row_counts.insert(table.clone(), count); + } + Ok(SchemaManifest { + revision: CATALOG_REVISION, + migration_version, + scoped_tables: live_tables.into_iter().collect(), + row_counts, + fenced_tables: fenced_tables.into_iter().collect(), + }) + } + + /// Freeze inventory and move submitted → inventoried atomically. + pub async fn freeze_inventory( + &self, + request_id: Uuid, + inventory: &FrozenInventory, + ) -> Result { + validate_storage_manifest(&inventory.storage)?; + let digest = inventory.digest()?; + let schema = serde_json::to_value(&inventory.schema)?; + let storage = serde_json::to_value(&inventory.storage)?; + let frozen = serde_json::to_value(inventory)?; + let row = sqlx::query( + r#" + UPDATE community_deletion_requests + SET stage = 'inventoried', schema_manifest = $2, storage_manifest = $3, + inventory_manifest = $4, inventory_digest = $5, + inventory_frozen_at = now(), updated_at = now(), + last_error = NULL, last_error_at = NULL + WHERE id = $1 AND stage = 'submitted' AND blocked_at IS NULL + RETURNING * + "#, + ) + .bind(request_id) + .bind(schema) + .bind(storage) + .bind(frozen) + .bind(digest) + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| { + DbError::InvalidData(format!( + "deletion {request_id} is not an unblocked submitted request" + )) + })?; + row_to_request(row) + } + + /// Approve the exact frozen inventory and move inventoried → approved. + pub async fn approve( + &self, + request_id: Uuid, + approved_by: &str, + note: Option<&str>, + ) -> Result { + let mut tx = self.pool.begin().await?; + let digest: Vec = sqlx::query_scalar( + "SELECT inventory_digest FROM community_deletion_requests \ + WHERE id = $1 AND stage = 'inventoried' AND blocked_at IS NULL FOR UPDATE", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::InvalidData(format!( + "deletion {request_id} is not an unblocked inventoried request" + )) + })?; + sqlx::query( + "INSERT INTO community_deletion_approvals \ + (request_id, inventory_digest, approved_by, note) VALUES ($1, $2, $3, $4)", + ) + .bind(request_id) + .bind(&digest) + .bind(approved_by) + .bind(note) + .execute(&mut *tx) + .await?; + let row = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = 'approved', updated_at = now(), next_attempt_at = now() \ + WHERE id = $1 RETURNING *", + ) + .bind(request_id) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + row_to_request(row) + } + + /// Claim a specific runnable request. Expired claims may be reclaimed. + pub async fn claim_specific( + &self, + request_id: Uuid, + owner: &str, + lease_duration: Duration, + ) -> Result> { + self.claim(Some(request_id), owner, lease_duration).await + } + + /// Claim the oldest runnable request. Expired claims may be reclaimed. + pub async fn claim_next( + &self, + owner: &str, + lease_duration: Duration, + ) -> Result> { + self.claim(None, owner, lease_duration).await + } + + async fn claim( + &self, + request_id: Option, + owner: &str, + lease_duration: Duration, + ) -> Result> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let row = sqlx::query( + r#" + WITH candidate AS ( + SELECT id + FROM community_deletion_requests + WHERE ($1::uuid IS NULL OR id = $1) + AND stage IN ('approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified') + AND blocked_at IS NULL + AND next_attempt_at <= now() + AND (lease_until IS NULL OR lease_until < now()) + ORDER BY created_at, id + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + UPDATE community_deletion_requests request + SET lease_owner = $2, + lease_generation = request.lease_generation + 1, + lease_until = now() + make_interval(secs => $3), + attempts = request.attempts + 1, + updated_at = now() + FROM candidate + WHERE request.id = candidate.id + RETURNING request.* + "#, + ) + .bind(request_id) + .bind(owner) + .bind(lease_seconds) + .fetch_optional(&self.pool) + .await?; + row.map(|row| { + let request = row_to_request(row)?; + let lease = LeaseToken { + request_id: request.id, + owner: owner.to_owned(), + generation: request.lease_generation, + community_id: request.community_id, + fence_generation: request.fence_generation, + }; + Ok(ClaimedDeletion { request, lease }) + }) + .transpose() + } + + /// Renew an owned claim and persist executor liveness. + pub async fn heartbeat( + &self, + token: &LeaseToken, + executor_mode: &str, + lease_duration: Duration, + draining: bool, + ) -> Result<()> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET lease_until = now() + make_interval(secs => $4), updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3 \ + AND lease_until >= now()", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(lease_seconds) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(stale_lease_error(token)); + } + sqlx::query( + "INSERT INTO community_deletion_executor_heartbeats \ + (executor_id, mode, request_id, draining) VALUES ($1, $2, $3, $4) \ + ON CONFLICT (executor_id) DO UPDATE SET mode = EXCLUDED.mode, \ + request_id = EXCLUDED.request_id, heartbeat_at = now(), \ + draining = EXCLUDED.draining, stopped_at = NULL", + ) + .bind(&token.owner) + .bind(executor_mode) + .bind(token.request_id) + .bind(draining) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark an executor stopped and release its current claim if still owned. + pub async fn stop_executor(&self, token: Option<&LeaseToken>, executor_id: &str) -> Result<()> { + let mut tx = self.pool.begin().await?; + if let Some(token) = token { + sqlx::query( + "UPDATE community_deletion_requests \ + SET lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .execute(&mut *tx) + .await?; + } + sqlx::query( + "UPDATE community_deletion_executor_heartbeats \ + SET request_id = NULL, draining = true, heartbeat_at = now(), stopped_at = now() \ + WHERE executor_id = $1", + ) + .bind(executor_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Acquire the universal durable fence and move approved → fenced. + pub async fn fence(&self, token: &LeaseToken) -> Result { + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, DeletionStage::Approved).await?; + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let current_generation: i64 = sqlx::query_scalar( + "SELECT deletion_fence_generation FROM communities WHERE id = $1 FOR UPDATE", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + let generation = current_generation.checked_add(1).ok_or_else(|| { + DbError::InvalidData("community deletion fence generation overflow".to_string()) + })?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + sqlx::query( + "UPDATE communities SET deletion_state = 'fenced', \ + deletion_fence_generation = $2, archived_at = COALESCE(archived_at, now()) \ + WHERE id = $1 AND deletion_state = 'active'", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .execute(&mut *tx) + .await?; + advance_request_tx( + &mut tx, + token, + DeletionStage::Approved, + DeletionStage::Fenced, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Approved, + "activate_fence", + serde_json::json!({"fence_generation": generation}), + ) + .await?; + tx.commit().await?; + Ok(generation) + } + + /// Verify fence ownership and record that serving writes drained. + pub async fn mark_drained(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + advance_request_tx( + &mut tx, + token, + DeletionStage::Fenced, + DeletionStage::Drained, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Fenced, + "serving_writes_drained", + serde_json::json!({"fence_generation": generation}), + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Mark storage binding removal after adapter verification. + pub async fn mark_bindings_removed( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + self.advance_with_checkpoint( + token, + DeletionStage::Drained, + DeletionStage::BindingsRemoved, + "remove_storage_bindings", + detail, + ) + .await + } + + /// Purge every scoped PostgreSQL table, preserve the community tombstone, and + /// move bindings_removed → postgres_purged in one transaction. + pub async fn purge_postgres(&self, token: &LeaseToken) -> Result> { + let generation = require_fence_generation(token)?; + // Re-inventory before opening the purge transaction. Any drift blocks; + // the transaction then locks the control and tombstone rows and all + // tenant writes are already fenced. + self.inventory_schema(token.community_id).await?; + + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::BindingsRemoved, generation).await?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + + let mut deleted = BTreeMap::new(); + // Deleting channels cascades some child tables. Alphabetical execution + // plus per-table WHERE is safe and idempotent; later units simply see 0. + for table in PURGE_SCOPED_TABLES { + let sql = format!("DELETE FROM {table} WHERE community_id = $1"); + let affected = sqlx::query(AssertSqlSafe(sql)) + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await? + .rows_affected(); + deleted.insert((*table).to_owned(), affected); + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + &format!("purge:{table}"), + serde_json::json!({"rows": affected}), + ) + .await?; + } + + sqlx::query( + "UPDATE communities SET deletion_state = 'tombstone', \ + deleted_at = COALESCE(deleted_at, now()), \ + archived_at = COALESCE(archived_at, now()), \ + signing_key = NULL, icon = NULL \ + WHERE id = $1 AND deletion_state = 'fenced' \ + AND deletion_fence_generation = $2", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .execute(&mut *tx) + .await?; + advance_request_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + DeletionStage::PostgresPurged, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::BindingsRemoved, + "postgres_tombstone_committed", + serde_json::to_value(&deleted)?, + ) + .await?; + tx.commit().await?; + Ok(deleted) + } + + /// Mark cache purge after Redis adapter verification. + pub async fn mark_cache_purged( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + self.advance_with_checkpoint( + token, + DeletionStage::PostgresPurged, + DeletionStage::CachePurged, + "purge_cache_namespace", + detail, + ) + .await + } + + /// Verify PostgreSQL logical absence without advancing the cross-store stage. + /// + /// The caller must verify object storage and Redis too, then call + /// [`Self::mark_logically_verified`]. Keeping the transition separate makes + /// a crash after any partial verification safely repeat the whole proof. + pub async fn verify_postgres_logically_deleted(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::CachePurged, generation).await?; + let tombstone: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 \ + AND deletion_state = 'tombstone' AND deleted_at IS NOT NULL \ + AND deletion_fence_generation = $2)", + ) + .bind(token.community_id.as_uuid()) + .bind(generation) + .fetch_one(&mut *tx) + .await?; + if !tombstone { + return Err(DbError::InvalidData(format!( + "community {} tombstone/fence verification failed", + token.community_id + ))); + } + for table in EXPECTED_SCOPED_TABLES { + let sql = + format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE community_id = $1 LIMIT 1)"); + let remains: bool = sqlx::query_scalar(AssertSqlSafe(sql)) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if remains { + return Err(DbError::InvalidData(format!( + "logical verification found tenant rows in {table}" + ))); + } + } + tx.commit().await?; + Ok(()) + } + + /// Commit the cross-store logical verification checkpoint. + pub async fn mark_logically_verified( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + self.advance_with_checkpoint( + token, + DeletionStage::CachePurged, + DeletionStage::LogicallyVerified, + "verify_cross_store_absence", + detail, + ) + .await + } + + /// Finish logical deletion and enter the physical-expiry pending state. + pub async fn mark_retention_pending( + &self, + token: &LeaseToken, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::LogicallyVerified, generation) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::LogicallyVerified, + "retention_physical_expiry_pending", + detail, + ) + .await?; + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = 'retention_pending', completed_at = now(), updated_at = now(), \ + lease_owner = NULL, lease_until = NULL, last_error = NULL, last_error_at = NULL \ + WHERE id = $1 AND stage = 'logically_verified' \ + AND lease_owner = $2 AND lease_generation = $3 AND lease_until >= now() \ + AND fence_generation = $4", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(generation) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(stale_lease_error(token)); + } + tx.commit().await?; + Ok(()) + } + + /// Persist a retryable unit failure and release the claim. + pub async fn record_retry( + &self, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, + retry_after: Duration, + ) -> Result<()> { + let bounded = bound_text(error, 4096); + let retry_seconds = i64::try_from(retry_after.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, stage).await?; + checkpoint_failed_tx(&mut tx, token, stage, unit_key, &bounded).await?; + sqlx::query( + "UPDATE community_deletion_requests \ + SET retry_count = retry_count + 1, last_error = $4, last_error_at = now(), \ + next_attempt_at = now() + make_interval(secs => $5), \ + lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(&bounded) + .bind(retry_seconds) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Persist a fail-closed permanent block and release the claim. + pub async fn block( + &self, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, + ) -> Result<()> { + let bounded = bound_text(error, 4096); + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, stage).await?; + checkpoint_failed_tx(&mut tx, token, stage, unit_key, &bounded).await?; + sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = now(), blocked_reason = $4, last_error = $4, \ + last_error_at = now(), lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(&bounded) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Take the shared community deletion lock inside an existing transaction. + pub async fn guard_transaction( + &self, + tx: &mut Transaction<'_, Postgres>, + community: CommunityId, + ) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(community.as_uuid()) + .execute(&mut **tx) + .await?; + let state: Option = sqlx::query_scalar( + "SELECT deletion_state FROM communities WHERE id = $1 AND deleted_at IS NULL", + ) + .bind(community.as_uuid()) + .fetch_optional(&mut **tx) + .await?; + match state.as_deref() { + Some("active") => Ok(()), + Some(other) => Err(DbError::AccessDenied(format!( + "community {community} is write-fenced ({other})" + ))), + None => Err(DbError::AccessDenied(format!( + "community {community} is missing or tombstoned" + ))), + } + } + + /// Begin a serving write and hold the community's shared deletion lock. + pub async fn begin_serving_write(&self, community: CommunityId) -> Result { + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(community.as_uuid()) + .execute(&mut *tx) + .await?; + let state: Option = sqlx::query_scalar( + "SELECT deletion_state FROM communities WHERE id = $1 AND deleted_at IS NULL", + ) + .bind(community.as_uuid()) + .fetch_optional(&mut *tx) + .await?; + match state.as_deref() { + Some("active") => Ok(ServingWriteGuard { _tx: tx }), + Some(other) => Err(DbError::AccessDenied(format!( + "community {community} is write-fenced ({other})" + ))), + None => Err(DbError::AccessDenied(format!( + "community {community} is missing or tombstoned" + ))), + } + } + + /// Ensure the durable community write fence allows serving writes. + pub async fn assert_serving_write_allowed(&self, community: CommunityId) -> Result<()> { + let guard = self.begin_serving_write(community).await?; + drop(guard); + Ok(()) + } + + /// Whether a community remains active and serving-write eligible. + pub async fn is_serving_active(&self, community: CommunityId) -> Result { + sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 \ + AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", + ) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + + async fn advance_with_checkpoint( + &self, + token: &LeaseToken, + from: DeletionStage, + to: DeletionStage, + unit_key: &str, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, from, generation).await?; + advance_request_tx(&mut tx, token, from, to, Some(generation)).await?; + checkpoint_completed_tx(&mut tx, token, from, unit_key, detail).await?; + tx.commit().await?; + Ok(()) + } + + async fn live_scoped_tables(&self) -> Result> { + let rows: Vec = sqlx::query_scalar( + r#" + SELECT c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition + AND a.attname = 'community_id' + AND NOT a.attisdropped + AND c.relname NOT IN ( + 'community_deletion_requests', + 'community_deletion_approvals', + 'community_deletion_checkpoints', + 'community_deletion_retention_exceptions', + 'community_deletion_executor_heartbeats' + ) + ORDER BY c.relname + "#, + ) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().collect()) + } + + async fn live_fenced_tables(&self) -> Result> { + let rows: Vec = sqlx::query_scalar( + r#" + SELECT c.relname + FROM pg_trigger trigger + JOIN pg_class c ON c.oid = trigger.tgrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid + WHERE n.nspname = 'public' + AND NOT trigger.tgisinternal + AND NOT c.relispartition + AND procedure.proname = 'enforce_community_write_fence' + ORDER BY c.relname + "#, + ) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().collect()) + } +} + +/// Fail closed when storage inventory reports unknown or unsupported data. +pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { + if manifest.version != 1 { + return Err(DbError::InvalidData(format!( + "unsupported storage manifest version {}", + manifest.version + ))); + } + if !manifest.unknown_keys.is_empty() { + return Err(DbError::InvalidData(format!( + "unknown object-store keys block deletion: {}", + manifest.unknown_keys.join(",") + ))); + } + if !manifest.unsupported_version_keys.is_empty() { + return Err(DbError::InvalidData(format!( + "unsupported object versions block deletion: {}", + manifest.unsupported_version_keys.join(",") + ))); + } + Ok(()) +} + +async fn verify_lease( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, +) -> Result<()> { + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ + WHERE id = $1 AND stage = $2 AND lease_owner = $3 \ + AND lease_generation = $4 AND lease_until >= now() AND blocked_at IS NULL)", + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(&token.owner) + .bind(token.generation) + .fetch_one(&mut **tx) + .await?; + if valid { + Ok(()) + } else { + Err(stale_lease_error(token)) + } +} + +async fn verify_lease_and_fence( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + fence_generation: i64, +) -> Result<()> { + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests request \ + JOIN communities community ON community.id = request.community_id \ + WHERE request.id = $1 AND request.stage = $2 AND request.lease_owner = $3 \ + AND request.lease_generation = $4 AND request.lease_until >= now() \ + AND request.blocked_at IS NULL AND request.fence_generation = $5 \ + AND community.deletion_state IN ('fenced', 'tombstone') \ + AND community.deletion_fence_generation = $5)", + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(&token.owner) + .bind(token.generation) + .bind(fence_generation) + .fetch_one(&mut **tx) + .await?; + if valid { + Ok(()) + } else { + Err(DbError::AccessDenied(format!( + "stale lease or fencing generation for deletion {}", + token.request_id + ))) + } +} + +async fn set_executor_gucs( + tx: &mut Transaction<'_, Postgres>, + community: CommunityId, + generation: i64, +) -> Result<()> { + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', $2, true)", + ) + .bind(community.to_string()) + .bind(generation.to_string()) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn advance_request_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + from: DeletionStage, + to: DeletionStage, + fence_generation: Option, +) -> Result<()> { + if from.next() != Some(to) { + return Err(DbError::InvalidData(format!( + "illegal deletion transition {from} -> {to}" + ))); + } + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET stage = $5, fence_generation = COALESCE($6, fence_generation), \ + updated_at = now(), last_error = NULL, last_error_at = NULL \ + WHERE id = $1 AND stage = $4 AND lease_owner = $2 \ + AND lease_generation = $3 AND lease_until >= now() AND blocked_at IS NULL", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(from.to_string()) + .bind(to.to_string()) + .bind(fence_generation) + .execute(&mut **tx) + .await? + .rows_affected(); + if affected == 1 { + Ok(()) + } else { + Err(stale_lease_error(token)) + } +} + +async fn checkpoint_completed_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + detail: serde_json::Value, +) -> Result<()> { + sqlx::query( + r#" + INSERT INTO community_deletion_checkpoints + (request_id, stage, unit_key, status, lease_generation, detail, completed_at) + VALUES ($1, $2, $3, 'completed', $4, $5, now()) + ON CONFLICT (request_id, stage, unit_key) DO UPDATE + SET status = 'completed', lease_generation = EXCLUDED.lease_generation, + attempts = community_deletion_checkpoints.attempts + 1, + detail = EXCLUDED.detail, error = NULL, completed_at = now() + "#, + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(unit_key) + .bind(token.generation) + .bind(detail) + .execute(&mut **tx) + .await?; + Ok(()) +} + +async fn checkpoint_failed_tx( + tx: &mut Transaction<'_, Postgres>, + token: &LeaseToken, + stage: DeletionStage, + unit_key: &str, + error: &str, +) -> Result<()> { + sqlx::query( + r#" + INSERT INTO community_deletion_checkpoints + (request_id, stage, unit_key, status, lease_generation, error) + VALUES ($1, $2, $3, 'failed', $4, $5) + ON CONFLICT (request_id, stage, unit_key) DO UPDATE + SET status = 'failed', lease_generation = EXCLUDED.lease_generation, + attempts = community_deletion_checkpoints.attempts + 1, + error = EXCLUDED.error, completed_at = NULL + "#, + ) + .bind(token.request_id) + .bind(stage.to_string()) + .bind(unit_key) + .bind(token.generation) + .bind(error) + .execute(&mut **tx) + .await?; + Ok(()) +} + +fn serialize_community_id( + community: &CommunityId, + serializer: S, +) -> std::result::Result +where + S: serde::Serializer, +{ + serializer.serialize_str(&community.to_string()) +} + +fn row_to_request(row: sqlx::postgres::PgRow) -> Result { + let community_id: Uuid = row.try_get("community_id")?; + let digest: Option> = row.try_get("inventory_digest")?; + Ok(DeletionRequest { + id: row.try_get("id")?, + community_id: CommunityId::from_uuid(community_id), + community_host: row.try_get("community_host")?, + stage: row.try_get::("stage")?.parse()?, + requested_by: row.try_get("requested_by")?, + reason: row.try_get("reason")?, + schema_manifest: row.try_get("schema_manifest")?, + storage_manifest: row.try_get("storage_manifest")?, + inventory_manifest: row.try_get("inventory_manifest")?, + inventory_digest: digest.map(hex::encode), + fence_generation: row.try_get("fence_generation")?, + lease_owner: row.try_get("lease_owner")?, + lease_generation: row.try_get("lease_generation")?, + lease_until: row.try_get("lease_until")?, + attempts: row.try_get("attempts")?, + retry_count: row.try_get("retry_count")?, + last_error: row.try_get("last_error")?, + blocked_reason: row.try_get("blocked_reason")?, + created_at: row.try_get("created_at")?, + updated_at: row.try_get("updated_at")?, + completed_at: row.try_get("completed_at")?, + }) +} + +fn stale_lease_error(token: &LeaseToken) -> DbError { + DbError::AccessDenied(format!( + "stale deletion lease {} owner {:?} generation {}", + token.request_id, token.owner, token.generation + )) +} + +fn require_fence_generation(token: &LeaseToken) -> Result { + token.fence_generation.ok_or_else(|| { + DbError::InvalidData(format!( + "deletion {} has no durable fence generation", + token.request_id + )) + }) +} + +fn bound_text(input: &str, max: usize) -> String { + if input.len() <= max { + return input.to_owned(); + } + let mut end = max; + while !input.is_char_boundary(end) { + end -= 1; + } + input[..end].to_owned() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn storage_manifest() -> StorageManifest { + StorageManifest { + version: 1, + tenant_keys: Vec::new(), + git_pointer_keys: Vec::new(), + media_sidecar_keys: Vec::new(), + media_upload_keys: Vec::new(), + retained_shared_cas_keys: Vec::new(), + unknown_keys: Vec::new(), + unsupported_version_keys: Vec::new(), + } + } + + #[test] + fn stage_order_is_exact_and_terminal() { + let mut stage = DeletionStage::Submitted; + let mut seen = vec![stage]; + while let Some(next) = stage.next() { + stage = next; + seen.push(stage); + } + assert_eq!( + seen, + vec![ + DeletionStage::Submitted, + DeletionStage::Inventoried, + DeletionStage::Approved, + DeletionStage::Fenced, + DeletionStage::Drained, + DeletionStage::BindingsRemoved, + DeletionStage::PostgresPurged, + DeletionStage::CachePurged, + DeletionStage::LogicallyVerified, + DeletionStage::RetentionPending, + ] + ); + assert!(!DeletionStage::Submitted.runnable()); + assert!(!DeletionStage::Inventoried.runnable()); + assert!(DeletionStage::Approved.runnable()); + assert!(!DeletionStage::RetentionPending.runnable()); + } + + #[test] + fn unknown_storage_data_fails_closed() { + let mut manifest = storage_manifest(); + manifest.unknown_keys.push("mystery/data".to_string()); + let error = validate_storage_manifest(&manifest).expect_err("unknown key must block"); + assert!(error.to_string().contains("unknown object-store keys")); + } + + #[test] + fn unsupported_object_versions_fail_closed() { + let mut manifest = storage_manifest(); + manifest + .unsupported_version_keys + .push("_meta/community/blob.json".to_string()); + let error = validate_storage_manifest(&manifest).expect_err("object version must block"); + assert!(error.to_string().contains("unsupported object versions")); + } + + #[test] + fn frozen_inventory_digest_is_stable() { + let inventory = FrozenInventory { + schema: SchemaManifest { + revision: 1, + migration_version: 27, + scoped_tables: vec!["events".to_string()], + row_counts: BTreeMap::from([("events".to_string(), 3)]), + fenced_tables: vec!["events".to_string()], + }, + storage: storage_manifest(), + }; + assert_eq!(inventory.digest().unwrap(), inventory.digest().unwrap()); + assert_eq!(inventory.digest().unwrap().len(), 32); + } + + #[test] + fn errors_are_utf8_bounded() { + let input = format!("{}🛸", "x".repeat(4095)); + let bounded = bound_text(&input, 4096); + assert!(bounded.len() <= 4096); + assert!(std::str::from_utf8(bounded.as_bytes()).is_ok()); + } +} + +#[cfg(test)] +mod postgres_tests { + use super::*; + use crate::{CreateCommunityWithOwnerResult, Db, DbConfig}; + + async fn store() -> (Db, DeletionStore) { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + let db = Db::new(&DbConfig { + database_url, + max_connections: 5, + min_connections: 0, + ..DbConfig::default() + }) + .await + .expect("connect deletion test DB"); + db.migrate().await.expect("migrate deletion test DB"); + let store = db.deletion_store(); + (db, store) + } + + async fn inventoried_request( + db: &Db, + store: &DeletionStore, + ) -> (DeletionRequest, FrozenInventory) { + let host = format!("deletion-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("create community"); + let submitted = store + .submit(&host, "test-operator", Some("test deletion")) + .await + .expect("submit"); + assert_eq!(submitted.community_id, community.id); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community.id) + .await + .expect("schema inventory"), + storage: StorageManifest { + version: 1, + tenant_keys: Vec::new(), + git_pointer_keys: Vec::new(), + media_sidecar_keys: Vec::new(), + media_upload_keys: Vec::new(), + retained_shared_cas_keys: Vec::new(), + unknown_keys: Vec::new(), + unsupported_version_keys: Vec::new(), + }, + }; + let request = store + .freeze_inventory(submitted.id, &inventory) + .await + .expect("freeze inventory"); + (request, inventory) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn approval_boundary_blocks_claim_until_exact_inventory_is_approved() { + let (db, store) = store().await; + let (request, inventory) = inventoried_request(&db, &store).await; + assert_eq!(request.stage, DeletionStage::Inventoried); + assert!(store + .claim_specific(request.id, "executor-a", DEFAULT_LEASE_DURATION) + .await + .expect("claim before approval") + .is_none()); + + let approved = store + .approve(request.id, "approver-a", Some("reviewed")) + .await + .expect("approve"); + assert_eq!(approved.stage, DeletionStage::Approved); + assert_eq!( + approved.inventory_digest, + Some(hex::encode(inventory.digest().unwrap())) + ); + assert!(store + .claim_specific(request.id, "executor-a", DEFAULT_LEASE_DURATION) + .await + .expect("claim approved") + .is_some()); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn stale_claim_and_fence_generation_fail_closed() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let mut stale = claim.lease.clone(); + stale.generation -= 1; + assert!( + store.fence(&stale).await.is_err(), + "stale lease must reject" + ); + + let generation = store.fence(&claim.lease).await.expect("fence"); + let mut wrong_fence = claim.lease.clone(); + wrong_fence.fence_generation = Some(generation + 1); + assert!( + store.mark_drained(&wrong_fence).await.is_err(), + "wrong fence generation must reject" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn fence_waits_for_open_write_and_rejects_it_after_transition() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + + let mut open_write = db + .begin_transaction() + .await + .expect("open write transaction"); + sqlx::query("INSERT INTO pubkey_allowlist (community_id, pubkey) VALUES ($1, $2)") + .bind(request.community_id.as_uuid()) + .bind(vec![7_u8; 32]) + .execute(&mut *open_write) + .await + .expect("write acquires shared deletion lock"); + + let store_for_fence = store.clone(); + let lease = claim.lease.clone(); + let fencing = tokio::spawn(async move { store_for_fence.fence(&lease).await }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !fencing.is_finished(), + "exclusive fence must wait for open writer" + ); + open_write + .commit() + .await + .expect("pre-fence writer commits first"); + fencing.await.expect("fence task").expect("fence completes"); + + assert!( + db.add_to_allowlist(request.community_id, &[8_u8; 32], &[9_u8; 32], None) + .await + .is_err(), + "post-fence serving write must fail" + ); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_guard_blocks_fence_until_external_side_effect_finishes() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let guard = store + .begin_serving_write(request.community_id) + .await + .expect("serving write guard"); + let store_for_fence = store.clone(); + let lease = claim.lease.clone(); + let fencing = tokio::spawn(async move { store_for_fence.fence(&lease).await }); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!( + !fencing.is_finished(), + "fence must wait for external side effect guard" + ); + drop(guard); + fencing.await.expect("fence task").expect("fence completes"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn checkpointed_resume_is_idempotent_and_tombstone_blocks_name_reuse() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + let host = request.community_host.clone(); + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + store.mark_drained(&token).await.expect("drain"); + store + .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) + .await + .expect("bindings"); + let first = store.purge_postgres(&token).await.expect("purge postgres"); + assert_eq!(first.len(), EXPECTED_SCOPED_TABLES.len()); + assert!( + store.purge_postgres(&token).await.is_err(), + "completed stage cannot be replayed under stale checkpoint state" + ); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache"); + store + .verify_postgres_logically_deleted(&token) + .await + .expect("logical postgres verify"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("mark verified"); + store + .mark_retention_pending(&token, serde_json::json!({"shared_cas": "retained"})) + .await + .expect("terminal"); + + let terminal = store.get(request.id).await.expect("terminal request"); + assert_eq!(terminal.stage, DeletionStage::RetentionPending); + let recreated = db + .create_community_with_owner( + &host, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + .await + .expect("recreate attempt"); + assert_eq!(recreated, CreateCommunityWithOwnerResult::HostExists); + assert!(db + .lookup_community_by_host_for_management(&host) + .await + .expect("tombstone lookup") + .is_some()); + assert!(db + .lookup_community_by_host(&host) + .await + .expect("serving lookup") + .is_none()); + } +} diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 9b26876747..8022472eac 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -17,6 +17,8 @@ pub mod api_token; pub mod archived_identities; /// Channel and membership persistence. pub mod channel; +/// Durable whole-community deletion lifecycle and PostgreSQL adapter. +pub mod deletion; /// Direct message channel persistence. pub mod dm; /// Database error types. @@ -1180,6 +1182,11 @@ impl Db { usage::community_hosts(&self.pool).await } + /// Return the shared durable whole-community deletion adapter. + pub fn deletion_store(&self) -> deletion::DeletionStore { + deletion::DeletionStore::new(self.pool.clone()) + } + /// Begin a database transaction for atomic multi-statement operations. /// /// Returns a `'static` transaction because `PgPool` is `Arc`-backed internally. @@ -1202,6 +1209,8 @@ impl Db { FROM communities WHERE lower(host) = lower($1) AND archived_at IS NULL + AND deleted_at IS NULL + AND deletion_state = 'active' "#, ) .bind(normalized_host) @@ -1223,7 +1232,7 @@ impl Db { /// Returns whether a community id still exists in the active lifecycle state. pub async fn is_community_active(&self, community_id: CommunityId) -> Result { let active = sqlx::query_scalar::<_, bool>( - "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 AND archived_at IS NULL)", + "SELECT EXISTS(SELECT 1 FROM communities WHERE id = $1 AND archived_at IS NULL AND deleted_at IS NULL AND deletion_state = 'active')", ) .bind(community_id.as_uuid()) .fetch_one(&self.pool) @@ -1305,6 +1314,8 @@ impl Db { FROM communities WHERE id = $1 AND archived_at IS NULL + AND deleted_at IS NULL + AND deletion_state = 'active' "#, ) .bind(community_id.as_uuid()) diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 37f54d0fa2..b1817c8e15 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -348,6 +348,11 @@ mod tests { "push_gateway_delivery_request_replays", "product_feedback", "replica_heartbeat", + "community_deletion_requests", + "community_deletion_approvals", + "community_deletion_checkpoints", + "community_deletion_retention_exceptions", + "community_deletion_executor_heartbeats", ] { if normalized[insert_pos..].contains(&format!("'{value}'")) { globals.insert(value.to_owned()); @@ -561,7 +566,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 28); + assert_eq!(migrations.len(), 29); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -919,6 +924,53 @@ mod tests { assert!(heartbeat.contains("epoch")); assert!(heartbeat.contains("INSERT INTO replica_heartbeat (id) VALUES (1)")); assert!(heartbeat.contains("_operator_global_tables")); + + + // Channel-id lookup index (0027): serves the tenant-independent + // `channels` lookups that carry no community_id predicate, which no + // community_id-leading index can satisfy. Covering + partial so the + // planner can go index-only; asserted NOT UNIQUE because `id` alone is + // not unique across communities. + let channel_id_index = migrations + .iter() + .find(|migration| migration.sql.as_str().contains("idx_channels_id_live")) + .expect("channel-id lookup migration"); + assert_eq!(channel_id_index.version, 27); + assert!(channel_id_index.sql.as_str().contains("INCLUDE (community_id)")); + assert!(channel_id_index.sql.as_str().contains("WHERE deleted_at IS NULL")); + assert!(!channel_id_index.sql.as_str().contains("CREATE UNIQUE INDEX")); + assert!(desired_schema.contains("idx_channels_id_live")); + + // Main already owns migration 0028 for long reaction payloads. + let long_reactions = migrations + .iter() + .find(|migration| { + migration + .sql + .as_str() + .contains("ALTER TABLE reactions ALTER COLUMN emoji TYPE VARCHAR(66)") + }) + .expect("long-reaction migration"); + assert_eq!(long_reactions.version, 28); + assert!(desired_schema.contains("emoji VARCHAR(66) NOT NULL")); + + // Durable whole-community deletion control plane. Its final migration + // number is reconciled after the branch is replayed onto current main. + let deletion = migrations + .iter() + .find(|migration| { + migration + .sql + .as_str() + .contains("CREATE TABLE community_deletion_requests") + }) + .expect("community-deletion migration"); + assert!(deletion.sql.as_str().contains("CREATE TABLE community_deletion_approvals")); + assert!(deletion.sql.as_str().contains("CREATE TABLE community_deletion_checkpoints")); + assert!(deletion.sql.as_str().contains("CREATE FUNCTION enforce_community_write_fence")); + assert!(deletion.sql.as_str().contains("CREATE FUNCTION enforce_community_tombstone")); + assert!(deletion.sql.as_str().contains("community tombstones are permanent")); + // Channel-id lookup index (0027): serves the tenant-independent // `channels` lookups that carry no community_id predicate, which no // community_id-leading index can satisfy. Covering + partial so the @@ -939,6 +991,7 @@ mod tests { desired_schema.contains("idx_channels_id_live"), "desired-state schema must carry the channel-id lookup index", ); +<<<<<<< HEAD assert_eq!(migrations[27].version, 28); let long_reactions = migrations[27].sql.as_str(); @@ -946,6 +999,26 @@ mod tests { long_reactions.contains("ALTER TABLE reactions ALTER COLUMN emoji TYPE VARCHAR(66)") ); assert!(desired_schema.contains("emoji VARCHAR(66) NOT NULL")); +||||||| parent of 037fcbdab (feat: add durable community deletion worker) +======= +||||||| parent of d04e4b503 (feat: add durable community deletion worker) +======= + + // Durable whole-community deletion control plane and universal DB fence. + assert_eq!(migrations[26].version, 27); + let deletion = migrations[26].sql.as_str(); + assert!(deletion.contains("CREATE TABLE community_deletion_requests")); + assert!(deletion.contains("CREATE TABLE community_deletion_approvals")); + assert!(deletion.contains("CREATE TABLE community_deletion_checkpoints")); + assert!(deletion.contains("CREATE TABLE community_deletion_retention_exceptions")); + assert!(deletion.contains("CREATE TABLE community_deletion_executor_heartbeats")); + assert!(deletion.contains("CREATE FUNCTION enforce_community_write_fence")); + assert!(deletion.contains("CREATE FUNCTION enforce_community_tombstone")); + assert!(deletion.contains("community tombstones are permanent")); + assert!(deletion.contains("_operator_global_tables")); + assert!(deletion.contains("'submitted', 'inventoried', 'approved', 'fenced', 'drained'")); +>>>>>>> d04e4b503 (feat: add durable community deletion worker) +>>>>>>> 037fcbdab (feat: add durable community deletion worker) } #[test] diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml new file mode 100644 index 0000000000..6dc812cf4b --- /dev/null +++ b/crates/buzz-deletion/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "buzz-deletion" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true +description = "Durable whole-community deletion engine for Buzz" + +[dependencies] +anyhow = { workspace = true } +axum = { workspace = true } +buzz-core = { workspace = true } +buzz-db = { workspace = true } +buzz-media = { workspace = true } +clap = { version = "4", features = ["derive"] } +deadpool-redis = { workspace = true } +redis = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +tokio-util = { workspace = true } +uuid = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs new file mode 100644 index 0000000000..2c1aeb11c7 --- /dev/null +++ b/crates/buzz-deletion/src/lib.rs @@ -0,0 +1,752 @@ +#![deny(unsafe_code)] +#![warn(missing_docs)] +//! Shared durable whole-community deletion engine and store adapters. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use anyhow::{Context, Result}; +use buzz_db::deletion::{ + ClaimedDeletion, DeletionRequest, DeletionStage, DeletionStore, FrozenInventory, LeaseToken, + StorageManifest, DEFAULT_LEASE_DURATION, +}; +use buzz_db::{Db, DbConfig}; +use buzz_media::{deletion_inventory, CurrentObjectVersion, MediaStorage}; +use clap::Subcommand; +use serde::Serialize; +use tokio_util::sync::CancellationToken; +use uuid::Uuid; + +const DEFAULT_STORAGE_OBJECT_CAP: u64 = 1_000_000; +const WORKER_IDLE_POLL: Duration = Duration::from_secs(5); +const RETRY_DELAY: Duration = Duration::from_secs(30); + +/// Return the shared durable deletion store for relay and operator paths. +pub fn store(db: &Db) -> DeletionStore { + db.deletion_store() +} + +/// CLI-only whole-community deletion commands. +#[derive(Subcommand)] +pub enum Command { + /// Persist a deletion request and freeze its initial cross-store inventory. + Submit { + /// Canonical community host. Defaults to RELAY_URL's authority. + #[arg(long)] + host: Option, + /// Operator identity recorded on the request. + #[arg(long)] + requested_by: String, + /// Optional reason for the request. + #[arg(long)] + reason: Option, + }, + /// List deletion requests as JSON. + List { + /// Maximum records. + #[arg(long, default_value_t = 100, value_parser = clap::value_parser!(u16).range(1..=1000))] + limit: u16, + }, + /// Inspect one request, including approval/checkpoints/errors. + Inspect { + /// Deletion request UUID. + id: Uuid, + }, + /// Explicitly approve the exact frozen inventory digest. + Approve { + /// Deletion request UUID. + id: Uuid, + /// Approving operator identity. + #[arg(long)] + approved_by: String, + /// Optional approval note. + #[arg(long)] + note: Option, + }, + /// Claim and run one request until terminal/blocked. + Run { + /// Deletion request UUID. + id: Uuid, + /// Executor identity (defaults to hostname/pid). + #[arg(long)] + executor_id: Option, + }, + /// Drain the currently runnable deletion queue, then exit. + Drain { + /// Executor identity (defaults to hostname/pid). + #[arg(long)] + executor_id: Option, + }, + /// Poll continuously in a dedicated no-ingress worker process. + Worker { + /// Executor identity (defaults to hostname/pid). + #[arg(long)] + executor_id: Option, + }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LoopMode { + Run, + Drain, + Worker, +} + +impl LoopMode { + const fn as_str(self) -> &'static str { + match self { + Self::Run => "run", + Self::Drain => "drain", + Self::Worker => "worker", + } + } +} + +struct Services { + store: DeletionStore, + media: Arc, + redis: deadpool_redis::Pool, +} + +#[derive(Debug, Serialize)] +struct RunOutput { + request_id: Uuid, + stage: DeletionStage, + blocked_reason: Option, +} + +/// Execute one nested deletion command. +pub async fn run(command: Command) -> Result { + let services = connect_services().await?; + match command { + Command::Submit { + host, + requested_by, + reason, + } => { + let host = host.unwrap_or_else(|| { + buzz_core::tenant::relay_url_authority( + &std::env::var("RELAY_URL") + .unwrap_or_else(|_| "ws://localhost:3000".to_string()), + ) + }); + if host.is_empty() { + anyhow::bail!("cannot derive community host; pass --host or set RELAY_URL"); + } + let request = services + .store + .submit(&host, &requested_by, reason.as_deref()) + .await?; + let inventory = build_inventory(&services, &request).await?; + let request = services + .store + .freeze_inventory(request.id, &inventory) + .await?; + print_json(&request)?; + Ok(0) + } + Command::List { limit } => { + print_json(&services.store.list(i64::from(limit)).await?)?; + Ok(0) + } + Command::Inspect { id } => { + print_json(&services.store.inspect(id).await?)?; + Ok(0) + } + Command::Approve { + id, + approved_by, + note, + } => { + print_json( + &services + .store + .approve(id, &approved_by, note.as_deref()) + .await?, + )?; + Ok(0) + } + Command::Run { id, executor_id } => { + run_loop( + services, + LoopMode::Run, + Some(id), + executor_id.unwrap_or_else(default_executor_id), + ) + .await + } + Command::Drain { executor_id } => { + run_loop( + services, + LoopMode::Drain, + None, + executor_id.unwrap_or_else(default_executor_id), + ) + .await + } + Command::Worker { executor_id } => { + run_loop( + services, + LoopMode::Worker, + None, + executor_id.unwrap_or_else(default_executor_id), + ) + .await + } + } +} + +async fn connect_services() -> Result { + let database_url = std::env::var("DATABASE_URL") + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + let db = Db::new(&DbConfig { + database_url, + max_connections: env_parse("BUZZ_DB_POOL_SIZE", 20), + ..DbConfig::default() + }) + .await?; + let store = store(&db); + let media_config = buzz_media::MediaConfig { + s3_endpoint: std::env::var("BUZZ_S3_ENDPOINT") + .unwrap_or_else(|_| "http://localhost:9000".to_string()), + s3_access_key: std::env::var("BUZZ_S3_ACCESS_KEY") + .unwrap_or_else(|_| "buzz_dev".to_string()), + s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY") + .unwrap_or_else(|_| "buzz_dev_secret".to_string()), + s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()), + s3_region: std::env::var("BUZZ_S3_REGION") + .or_else(|_| std::env::var("AWS_REGION")) + .unwrap_or_else(|_| "us-east-1".to_string()), + s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") + .unwrap_or_else(|_| "path".to_string()) + .parse() + .map_err(anyhow::Error::msg)?, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }; + let media = Arc::new(MediaStorage::new(&media_config)?); + let redis_url = + std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost:6379".to_string()); + let mut redis_config = deadpool_redis::Config::from_url(&redis_url); + redis_config.pool = Some(deadpool_redis::PoolConfig::new(env_parse( + "BUZZ_REDIS_POOL_SIZE", + 16, + ))); + let redis = redis_config + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .context("create deletion Redis pool")?; + Ok(Services { + store, + media, + redis, + }) +} + +fn env_parse(name: &str, default: T) -> T +where + T: std::str::FromStr, +{ + std::env::var(name) + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(default) +} + +async fn build_inventory( + services: &Services, + request: &DeletionRequest, +) -> Result { + let schema = services + .store + .inventory_schema(request.community_id) + .await?; + let storage = build_storage_manifest(services, request).await?; + Ok(FrozenInventory { schema, storage }) +} + +async fn build_storage_manifest( + services: &Services, + request: &DeletionRequest, +) -> Result { + let max_objects = storage_object_cap(); + let objects = services.media.list_all_for_deletion(max_objects).await?; + let bucket = deletion_inventory( + *request.community_id.as_uuid(), + objects + .iter() + .map(|object| (object.key.clone(), object.size)), + ); + let mut unsupported_version_keys = Vec::new(); + for key in bucket.tenant_keys() { + if matches!( + services.media.inspect_current_version(&key).await?, + CurrentObjectVersion::Present { + version_id: Some(_) + } + ) { + unsupported_version_keys.push(key); + } + } + let storage = StorageManifest { + version: 1, + tenant_keys: bucket.tenant_keys(), + git_pointer_keys: bucket.git_pointer_keys, + media_sidecar_keys: bucket.media_sidecar_keys, + media_upload_keys: bucket.media_upload_keys, + retained_shared_cas_keys: bucket.retained_shared_cas_keys, + unknown_keys: bucket.unknown_keys, + unsupported_version_keys, + }; + buzz_db::deletion::validate_storage_manifest(&storage)?; + Ok(storage) +} + +async fn run_loop( + services: Services, + mode: LoopMode, + request_id: Option, + executor_id: String, +) -> Result { + let shutdown = shutdown_token(); + let _health = if mode == LoopMode::Worker { + Some(spawn_worker_health(shutdown.clone()).await?) + } else { + None + }; + let mut ran = false; + loop { + if shutdown.is_cancelled() { + services + .store + .stop_executor(None, &executor_id) + .await + .context("record executor drain")?; + return Ok(0); + } + let claim = match request_id { + Some(id) => { + services + .store + .claim_specific(id, &executor_id, DEFAULT_LEASE_DURATION) + .await? + } + None => { + services + .store + .claim_next(&executor_id, DEFAULT_LEASE_DURATION) + .await? + } + }; + let Some(claim) = claim else { + match mode { + LoopMode::Worker => { + tokio::select! { + _ = shutdown.cancelled() => continue, + _ = tokio::time::sleep(WORKER_IDLE_POLL) => continue, + } + } + LoopMode::Run if !ran => anyhow::bail!( + "deletion request is not runnable, is blocked, or is leased by another executor" + ), + LoopMode::Run | LoopMode::Drain => return Ok(0), + } + }; + ran = true; + let output = execute_claim(&services, mode, claim, &shutdown).await?; + print_json(&output)?; + if mode == LoopMode::Run || shutdown.is_cancelled() { + return Ok(i32::from(output.blocked_reason.is_some())); + } + } +} + +async fn execute_claim( + services: &Services, + mode: LoopMode, + mut claim: ClaimedDeletion, + shutdown: &CancellationToken, +) -> Result { + let token = claim.lease.clone(); + loop { + if shutdown.is_cancelled() { + services + .store + .heartbeat(&token, mode.as_str(), DEFAULT_LEASE_DURATION, true) + .await?; + services + .store + .stop_executor(Some(&token), &token.owner) + .await?; + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + services + .store + .heartbeat(&token, mode.as_str(), DEFAULT_LEASE_DURATION, false) + .await?; + let stage_result = execute_stage(services, &claim).await; + match stage_result { + Ok(()) => {} + Err(error) => { + let message = format!("{error:#}"); + if is_permanent_failure(&message) { + services + .store + .block(&token, claim.request.stage, "stage", &message) + .await?; + } else { + services + .store + .record_retry(&token, claim.request.stage, "stage", &message, RETRY_DELAY) + .await?; + } + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + } + let request = services.store.get(token.request_id).await?; + if request.stage == DeletionStage::RetentionPending || request.blocked_reason.is_some() { + return Ok(run_output(request)); + } + claim.request = request.clone(); + claim.lease.fence_generation = request.fence_generation; + } +} + +async fn execute_stage(services: &Services, claim: &ClaimedDeletion) -> Result<()> { + let request = &claim.request; + let token = token_with_current_fence(&claim.lease, request); + match request.stage { + DeletionStage::Approved => { + // Rebuild inventory immediately before the first destructive stage + // and require byte-equivalence with the approved frozen inventory. + let live = build_inventory(services, request).await?; + let frozen: FrozenInventory = serde_json::from_value( + request + .inventory_manifest + .clone() + .context("approved request has no frozen inventory")?, + )?; + if live != frozen { + anyhow::bail!("approved inventory drifted before fencing"); + } + services.store.fence(&token).await?; + } + DeletionStage::Fenced => { + publish_disconnect_community(&services.redis, request.community_id).await?; + services.store.mark_drained(&token).await?; + } + DeletionStage::Drained => { + let storage: StorageManifest = serde_json::from_value( + request + .storage_manifest + .clone() + .context("request has no frozen storage manifest")?, + )?; + buzz_db::deletion::validate_storage_manifest(&storage)?; + // Re-run the exhaustive bucket taxonomy after fencing. Any key that + // appeared after approval is drift and blocks before a delete. + let live_storage = build_storage_manifest(services, request).await?; + if live_storage != storage { + anyhow::bail!("approved storage inventory drifted before binding removal"); + } + for key in &storage.tenant_keys { + match services.media.inspect_current_version(key).await? { + CurrentObjectVersion::Present { version_id: None } => {} + CurrentObjectVersion::Present { + version_id: Some(_), + } => anyhow::bail!("unsupported object version appeared after approval: {key}"), + CurrentObjectVersion::Missing => { + anyhow::bail!("approved object binding disappeared before deletion: {key}") + } + } + } + services.media.delete_bindings(&storage.tenant_keys).await?; + for key in &storage.tenant_keys { + if services.media.head(key).await? { + anyhow::bail!("object binding still exists after delete: {key}"); + } + } + services + .store + .mark_bindings_removed( + &token, + serde_json::json!({"deleted_keys": storage.tenant_keys.len()}), + ) + .await?; + } + DeletionStage::BindingsRemoved => { + services.store.purge_postgres(&token).await?; + } + DeletionStage::PostgresPurged => { + let deleted = purge_redis_namespace(&services.redis, request.community_id).await?; + services + .store + .mark_cache_purged(&token, serde_json::json!({"deleted_keys": deleted})) + .await?; + } + DeletionStage::CachePurged => { + services + .store + .verify_postgres_logically_deleted(&token) + .await?; + verify_storage_absence(services, request).await?; + verify_redis_absence(&services.redis, request.community_id).await?; + services + .store + .mark_logically_verified( + &token, + serde_json::json!({"postgres": true, "object_store": true, "redis": true}), + ) + .await?; + } + DeletionStage::LogicallyVerified => { + let frozen: FrozenInventory = serde_json::from_value( + request + .inventory_manifest + .clone() + .context("request has no frozen inventory")?, + )?; + services + .store + .mark_retention_pending( + &token, + serde_json::json!({ + "retained_shared_cas_keys": frozen.storage.retained_shared_cas_keys, + "policy": "member-erasure and fleet-wide shared-CAS GC are out of V1 scope" + }), + ) + .await?; + } + DeletionStage::Submitted | DeletionStage::Inventoried => { + anyhow::bail!("request has not crossed the explicit approval boundary") + } + DeletionStage::RetentionPending => {} + } + Ok(()) +} + +fn token_with_current_fence(token: &LeaseToken, request: &DeletionRequest) -> LeaseToken { + LeaseToken { + fence_generation: request.fence_generation, + ..token.clone() + } +} + +async fn verify_storage_absence(services: &Services, request: &DeletionRequest) -> Result<()> { + let storage: StorageManifest = serde_json::from_value( + request + .storage_manifest + .clone() + .context("request has no frozen storage manifest")?, + )?; + for key in &storage.tenant_keys { + if services.media.head(key).await? { + anyhow::bail!("logical verification found object binding: {key}"); + } + } + Ok(()) +} + +async fn publish_disconnect_community( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result<()> { + let mut connection = pool.get().await?; + let channel = format!("buzz:{community}:conn-control"); + let _: u64 = redis::cmd("PUBLISH") + .arg(channel) + .arg(r#"{"op":"DisconnectCommunity"}"#) + .query_async(&mut *connection) + .await?; + Ok(()) +} + +async fn purge_redis_namespace( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result { + let mut connection = pool.get().await?; + let pattern = format!("buzz:{community}:*"); + let mut cursor = 0u64; + let mut deleted = 0u64; + loop { + let (next, keys): (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(&pattern) + .arg("COUNT") + .arg(1000) + .query_async(&mut *connection) + .await?; + if !keys.is_empty() { + let count: u64 = redis::cmd("UNLINK") + .arg(&keys) + .query_async(&mut *connection) + .await?; + deleted = deleted.saturating_add(count); + } + if next == 0 { + break; + } + cursor = next; + } + Ok(deleted) +} + +async fn verify_redis_absence( + pool: &deadpool_redis::Pool, + community: buzz_core::CommunityId, +) -> Result<()> { + let mut connection = pool.get().await?; + let pattern = format!("buzz:{community}:*"); + let (_cursor, keys): (u64, Vec) = redis::cmd("SCAN") + .arg(0) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(1) + .query_async(&mut *connection) + .await?; + if keys.is_empty() { + Ok(()) + } else { + anyhow::bail!("logical verification found Redis key: {}", keys[0]) + } +} + +fn storage_object_cap() -> u64 { + std::env::var("BUZZ_DELETION_STORAGE_MAX_OBJECTS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_STORAGE_OBJECT_CAP) +} + +fn default_executor_id() -> String { + let hostname = std::env::var("HOSTNAME").unwrap_or_else(|_| "buzz-admin".to_string()); + format!("{hostname}:{}", std::process::id()) +} + +async fn spawn_worker_health(shutdown: CancellationToken) -> Result> { + let address = + std::env::var("BUZZ_DELETION_HEALTH_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".to_string()); + let listener = tokio::net::TcpListener::bind(&address) + .await + .with_context(|| format!("bind deletion worker health endpoint {address}"))?; + let draining = Arc::new(AtomicBool::new(false)); + let signal = Arc::clone(&draining); + let cancel = shutdown.clone(); + tokio::spawn(async move { + cancel.cancelled().await; + signal.store(true, Ordering::Relaxed); + }); + let router = axum::Router::new() + .route("/_liveness", axum::routing::get(|| async { "ok" })) + .route( + "/_readiness", + axum::routing::get({ + let draining = Arc::clone(&draining); + move || { + let draining = Arc::clone(&draining); + async move { + if draining.load(Ordering::Relaxed) { + (axum::http::StatusCode::SERVICE_UNAVAILABLE, "draining") + } else { + (axum::http::StatusCode::OK, "ready") + } + } + } + }), + ); + Ok(tokio::spawn(async move { + if let Err(error) = axum::serve(listener, router) + .with_graceful_shutdown(shutdown.cancelled_owned()) + .await + { + eprintln!("deletion worker health endpoint failed: {error}"); + } + })) +} + +fn shutdown_token() -> CancellationToken { + let token = CancellationToken::new(); + let signal = token.clone(); + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{signal as unix_signal, SignalKind}; + if let Ok(mut terminate) = unix_signal(SignalKind::terminate()) { + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = terminate.recv() => {}, + } + } else { + let _ = tokio::signal::ctrl_c().await; + } + } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; + } + signal.cancel(); + }); + token +} + +fn is_permanent_failure(message: &str) -> bool { + [ + "drift", + "unknown object-store keys", + "unsupported object", + "catalog", + "write-fence", + "approved inventory", + "tombstone/fence", + ] + .iter() + .any(|needle| message.contains(needle)) +} + +fn run_output(request: DeletionRequest) -> RunOutput { + RunOutput { + request_id: request.id, + stage: request.stage, + blocked_reason: request.blocked_reason, + } +} + +fn print_json(value: &impl Serialize) -> Result<()> { + println!("{}", serde_json::to_string_pretty(value)?); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn permanent_failures_are_narrow_and_fail_closed() { + assert!(is_permanent_failure("community deletion catalog drift")); + assert!(is_permanent_failure("unknown object-store keys")); + assert!(!is_permanent_failure("temporary Redis connection reset")); + } + + #[tokio::test] + async fn shutdown_token_can_interrupt_idle_worker_sleep() { + let token = CancellationToken::new(); + token.cancel(); + let woke = tokio::select! { + _ = token.cancelled() => true, + _ = tokio::time::sleep(Duration::from_secs(30)) => false, + }; + assert!(woke); + } +} diff --git a/crates/buzz-media/src/bucket_index.rs b/crates/buzz-media/src/bucket_index.rs index bb83dc517f..08dd97f50f 100644 --- a/crates/buzz-media/src/bucket_index.rs +++ b/crates/buzz-media/src/bucket_index.rs @@ -753,3 +753,150 @@ mod tests { ); } } + +/// Classify a full bucket listing for whole-community deletion. +/// +/// Unknown keys are reported globally rather than ignored: deleting a tenant +/// while the bucket contains a writer shape this binary does not understand is +/// unsafe. Tenant-owned sidecars/upload records are returned separately from +/// shared immutable blob/thumb and Git CAS data. +pub fn deletion_inventory( + community: Uuid, + objects: impl IntoIterator, +) -> DeletionBucketInventory { + let mut inventory = DeletionBucketInventory::default(); + let repo_prefix = format!("repos/{community}/"); + for (key, _size) in objects { + match classify_key(&key) { + KeyClass::Sidecar { + community: owner, .. + } if owner == community => inventory.media_sidecar_keys.push(key), + KeyClass::Auxiliary { + community: owner, .. + } if owner == community => inventory.media_upload_keys.push(key), + KeyClass::Sidecar { .. } + | KeyClass::Auxiliary { .. } + | KeyClass::Blob { .. } + | KeyClass::Thumb { .. } => {} + KeyClass::Unknown => { + if let Some(owner) = git_pointer_community(&key) { + if owner == community && key.starts_with(&repo_prefix) { + inventory.git_pointer_keys.push(key); + } + } else if is_known_git_shared_key(&key) || key.starts_with("probe/") { + // Known shared immutable CAS/probe data is deliberately + // outside the per-community frozen manifest. V1 removes + // only tenant-owned bindings; fleet-wide physical GC is a + // separate retention phase. + } else { + inventory.unknown_keys.push(key); + } + } + } + } + inventory.media_sidecar_keys.sort(); + inventory.media_upload_keys.sort(); + inventory.git_pointer_keys.sort(); + inventory.retained_shared_cas_keys.sort(); + inventory.unknown_keys.sort(); + inventory +} + +fn git_pointer_community(key: &str) -> Option { + let mut parts = key.split('/'); + if parts.next()? != "repos" { + return None; + } + let community = parse_canonical_uuid(parts.next()?)?; + let owner = parts.next()?; + let repo = parts.next()?; + let pointer = parts.next()?; + if parts.next().is_some() + || owner.len() != 64 + || !owner.bytes().all(|byte| byte.is_ascii_hexdigit()) + || repo.is_empty() + || repo.len() > 64 + || repo.starts_with('.') + || repo.contains("..") + || !repo + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-')) + || pointer != "pointer" + { + return None; + } + Some(community) +} + +fn is_known_git_shared_key(key: &str) -> bool { + ["packs/", "idx/", "manifests/"].iter().any(|prefix| { + key.strip_prefix(prefix).is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|byte| byte.is_ascii_hexdigit()) + }) + }) +} + +/// Object-store inventory relevant to one community deletion. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DeletionBucketInventory { + /// Community media serve-gate sidecars. + pub media_sidecar_keys: Vec, + /// Community upload/moderation records. + pub media_upload_keys: Vec, + /// Community Git repository pointers. + pub git_pointer_keys: Vec, + /// Shared immutable media/Git CAS data retained in V1. + pub retained_shared_cas_keys: Vec, + /// Bucket keys outside the exact writer taxonomy. + pub unknown_keys: Vec, +} + +impl DeletionBucketInventory { + /// Sorted union of all tenant-owned binding keys. + pub fn tenant_keys(&self) -> Vec { + let mut keys = self.media_sidecar_keys.clone(); + keys.extend(self.media_upload_keys.iter().cloned()); + keys.extend(self.git_pointer_keys.iter().cloned()); + keys.sort(); + keys + } +} + +#[cfg(test)] +mod deletion_inventory_tests { + use super::*; + + #[test] + fn inventories_target_bindings_and_ignores_shared_cas() { + let target = Uuid::from_u128(1); + let other = Uuid::from_u128(2); + let sha = "a".repeat(64); + let objects = vec![ + (format!("_meta/{target}/{sha}.json"), 1), + (format!("_meta/{other}/{sha}.json"), 1), + ( + format!("_uploads/{target}/{sha}/01ARZ3NDEKTSV4RRFFQ69G5FAV.json"), + 1, + ), + (format!("repos/{target}/{}/repo/pointer", "b".repeat(64)), 1), + (format!("packs/{sha}"), 1), + (format!("{sha}.png"), 1), + ]; + let inventory = deletion_inventory(target, objects); + assert_eq!(inventory.media_sidecar_keys.len(), 1); + assert_eq!(inventory.media_upload_keys.len(), 1); + assert_eq!(inventory.git_pointer_keys.len(), 1); + assert!(inventory.retained_shared_cas_keys.is_empty()); + assert!(inventory.unknown_keys.is_empty()); + assert_eq!(inventory.tenant_keys().len(), 3); + } + + #[test] + fn unknown_writer_shape_is_not_silently_skipped() { + let inventory = deletion_inventory( + Uuid::from_u128(1), + vec![("future-format/community/data".to_string(), 1)], + ); + assert_eq!(inventory.unknown_keys, vec!["future-format/community/data"]); + } +} diff --git a/crates/buzz-media/src/lib.rs b/crates/buzz-media/src/lib.rs index 67896d4ef2..0aeacfadde 100644 --- a/crates/buzz-media/src/lib.rs +++ b/crates/buzz-media/src/lib.rs @@ -14,12 +14,14 @@ pub mod upload_record; pub mod validation; pub use bucket_index::{ - classify_key, fold_bucket_listing, BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass, - Page, SweepError, + classify_key, deletion_inventory, fold_bucket_listing, BucketAggregate, BucketSnapshot, + CommunityStorage, DeletionBucketInventory, KeyClass, Page, SweepError, }; pub use config::{MediaConfig, S3AddressingStyle}; pub use error::MediaError; -pub use storage::{BlobHeadMeta, BlobMeta, ByteStream, MediaStorage}; +pub use storage::{ + BlobHeadMeta, BlobMeta, ByteStream, CurrentObjectVersion, DeletionObject, MediaStorage, +}; pub use types::BlobDescriptor; pub use upload::{process_file_upload, process_upload, process_video_upload}; pub use upload_record::{ diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index cbf980201f..fa5e54e6b1 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -177,6 +177,71 @@ impl MediaStorage { } } + /// Exhaustively list bucket objects for deletion inventory. + /// + /// Fails rather than returning a partial listing when pagination is + /// malformed or the explicit object cap is exceeded. + pub async fn list_all_for_deletion( + &self, + max_objects: u64, + ) -> Result, MediaError> { + let mut objects = Vec::new(); + let mut continuation = None; + loop { + let (result, _status) = self + .bucket + .list_page(String::new(), None, continuation.take(), None, Some(1000)) + .await?; + let next_len = objects.len().saturating_add(result.contents.len()); + if u64::try_from(next_len).unwrap_or(u64::MAX) > max_objects { + return Err(MediaError::StorageError(format!( + "deletion inventory exceeds object cap {max_objects}" + ))); + } + objects.extend(result.contents.into_iter().map(|object| DeletionObject { + key: object.key, + size: object.size, + last_modified: object.last_modified, + })); + if !result.is_truncated { + break; + } + continuation = result.next_continuation_token; + if continuation.is_none() { + return Err(MediaError::StorageError( + "truncated deletion inventory page has no continuation token".to_string(), + )); + } + } + Ok(objects) + } + + /// Return the current object version id, if the backend reports one. + /// + /// V1 refuses to delete versioned tenant bindings because rust-s3 cannot + /// exhaustively enumerate non-current versions; a plain delete marker would + /// not prove logical absence from storage. + pub async fn inspect_current_version( + &self, + key: &str, + ) -> Result { + match self.bucket.head_object(key).await { + Ok((result, _)) => Ok(CurrentObjectVersion::Present { + version_id: result.version_id, + }), + Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(CurrentObjectVersion::Missing), + Err(error) => Err(MediaError::StorageError(error.to_string())), + } + } + + /// Delete tenant binding keys one at a time and fail on any backend error. + pub async fn delete_bindings(&self, keys: &[String]) -> Result<(), MediaError> { + for key in keys { + self.delete(key).await?; + } + Ok(()) + } + /// Build the community-scoped sidecar key for a given sha256 (bare hash). /// /// Raw media bytes remain shared content-addressed CAS (`{sha}.{ext}`), but @@ -269,6 +334,29 @@ impl MediaStorage { } } +/// Current-version observation for a tenant-owned object binding. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CurrentObjectVersion { + /// The key does not exist. + Missing, + /// The key exists, optionally with a backend version id. + Present { + /// S3 version id. `Some` is unsupported for V1 deletion. + version_id: Option, + }, +} + +/// One object observed during exhaustive deletion inventory. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeletionObject { + /// Exact bucket key. + pub key: String, + /// Current object size. + pub size: u64, + /// Backend-reported last modification timestamp. + pub last_modified: String, +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/buzz-relay/Cargo.toml b/crates/buzz-relay/Cargo.toml index cbad2a3b29..314b533913 100644 --- a/crates/buzz-relay/Cargo.toml +++ b/crates/buzz-relay/Cargo.toml @@ -19,6 +19,7 @@ path = "src/main.rs" buzz-core = { workspace = true } buzz-conformance = { workspace = true } buzz-db = { workspace = true } +buzz-deletion = { workspace = true } buzz-auth = { workspace = true } buzz-pubsub = { workspace = true } buzz-audit = { workspace = true } diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index 53e3f59463..6f2eb81da5 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -1823,6 +1823,24 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { return response; } + // An already-running receive-pack may cross the durable fence after + // request admission. Revalidate immediately before object-store CAS; DB + // trigger fencing alone cannot roll back an S3 pointer mutation. + let _serving_write = match buzz_deletion::store(&state.db) + .begin_serving_write(ctx.tenant.community()) + .await + { + Ok(guard) => guard, + Err(error) => { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push rejected by community deletion fence"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community writes are fenced", + ) + .into_response(); + } + }; + // Step 7 (CAS). The PushContext binds `parent_state` (observed at // hydrate) to the CAS predicate here — no re-reading of the pointer // between hydrate and CAS. diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index fa0401bc26..41c10e8e63 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -310,6 +310,11 @@ pub async fn upload_blob( ) -> Result, MediaError> { let attribution = upload_attribution(&state, &auth, &headers).await; + let _serving_write = buzz_deletion::store(&state.db) + .begin_serving_write(auth.tenant.community()) + .await + .map_err(|_| MediaError::RelayMembershipRequired)?; + if auth.route_mode == UploadRouteMode::LegacyMedia { metrics::counter!("buzz_media_legacy_upload_route_total").increment(1); } diff --git a/crates/buzz-relay/src/handlers/command_executor.rs b/crates/buzz-relay/src/handlers/command_executor.rs index 2d82736807..85aae9d7b8 100644 --- a/crates/buzz-relay/src/handlers/command_executor.rs +++ b/crates/buzz-relay/src/handlers/command_executor.rs @@ -110,6 +110,12 @@ async fn persist_command_event( .begin_transaction() .await .map_err(|e| IngestError::Internal(format!("error: begin transaction: {e}")))?; + buzz_deletion::store(&state.db) + .guard_transaction(&mut tx, tenant.community()) + .await + .map_err(|error| { + IngestError::Rejected(format!("restricted: community writes are fenced: {error}")) + })?; // INSERT with ON CONFLICT DO NOTHING — idempotency guard. let id_bytes = event.id.as_bytes(); diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index a67797385b..c6d4655365 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -280,6 +280,18 @@ pub(crate) async fn fan_out_event_to_local_subscribers( /// Fan out one event received from Redis pub/sub to this relay's local subscribers. #[tracing::instrument(skip_all)] pub async fn fan_out_pubsub_event(state: &Arc, channel_event: buzz_pubsub::ChannelEvent) { + // Redis can carry an event published just before the deletion fence. Do not + // deliver stale post-fence fan-out to a community that is no longer active. + if !matches!( + state + .db + .is_community_active(channel_event.community_id) + .await, + Ok(true) + ) { + return; + } + // The Redis topic carries the tenant-local routing scope explicitly: // `Channel(id)` for a per-channel event, `Global` for a channel-less one. // Convert back to the `Option` channel id `fan_out()` indexes on — @@ -667,6 +679,22 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc guard, + Err(error) => { + reject("restricted"); + conn.send(RelayMessage::ok( + &event_id_hex, + false, + &format!("restricted: community writes are fenced: {error}"), + )); + return; + } + }; + if kind_u32 == buzz_core::kind::KIND_AUTH { reject("invalid"); conn.send(RelayMessage::ok( diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index cd9f20b5f4..de39169d68 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -1863,6 +1863,13 @@ async fn ingest_event_inner( let kind_u32 = event_kind_u32(&event); debug!(event_id = %event_id_hex, kind = kind_u32, "ingest_event"); + buzz_deletion::store(&state.db) + .assert_serving_write_allowed(tenant.community()) + .await + .map_err(|error| { + IngestError::Rejected(format!("restricted: community writes are fenced: {error}")) + })?; + if kind_u32 == KIND_AUTH { return Err(IngestError::Rejected( "invalid: AUTH events cannot be submitted".into(), diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 34dc2dfcf8..80fe5a678a 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -469,6 +469,7 @@ async fn main() -> anyhow::Result<()> { if let Some(handle) = buzz_relay::mesh_boot::boot_mesh( &state.config, state.redis_pool.clone(), + state.db.clone(), &state.relay_keypair, Arc::clone(&state.shutting_down), ) diff --git a/crates/buzz-relay/src/mesh_boot.rs b/crates/buzz-relay/src/mesh_boot.rs index 20e550aa08..4f452fed89 100644 --- a/crates/buzz-relay/src/mesh_boot.rs +++ b/crates/buzz-relay/src/mesh_boot.rs @@ -411,6 +411,7 @@ fn advertise_addrs(endpoint: &MeshEndpoint) -> Vec { pub async fn boot_mesh( config: &Config, redis_pool: deadpool_redis::Pool, + db: buzz_db::Db, relay_keypair: &nostr::Keys, shutting_down: Arc, ) -> anyhow::Result> { @@ -508,7 +509,7 @@ pub async fn boot_mesh( transport.set_inbound(Box::new(dispatcher.clone())); Ok(Some(MeshHandle { - directory: SessionDirectory::new(redis_pool), + directory: SessionDirectory::with_db(redis_pool, db), transport, membership: membership_arc, local_runtime_id: runtime_id, @@ -535,9 +536,18 @@ mod tests { .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .unwrap(); let keys = nostr::Keys::generate(); - let handle = boot_mesh(&config, pool, &keys, Arc::new(AtomicBool::new(false))) - .await - .expect("off path is never an error"); + let db_pool = sqlx::postgres::PgPoolOptions::new() + .connect_lazy("postgres://buzz:buzz_dev@localhost:5432/buzz") + .unwrap(); + let handle = boot_mesh( + &config, + pool, + buzz_db::Db::from_pool(db_pool), + &keys, + Arc::new(AtomicBool::new(false)), + ) + .await + .expect("off path is never an error"); assert!(handle.is_none()); } diff --git a/crates/buzz-relay/src/push_runtime.rs b/crates/buzz-relay/src/push_runtime.rs index 49845067ea..5a261ac912 100644 --- a/crates/buzz-relay/src/push_runtime.rs +++ b/crates/buzz-relay/src/push_runtime.rs @@ -418,6 +418,20 @@ async fn deliver_one( return; } }; + let _serving_write = match buzz_deletion::store(&state.db) + .begin_serving_write(outcome.community) + .await + { + Ok(guard) => guard, + Err(error) => { + warn!(wake=%outcome.id, %error, "push delivery suppressed by community deletion fence"); + let _ = state + .db + .fail_push_wake(outcome.community, outcome.id, outcome.claim_id) + .await; + return; + } + }; let Some(url) = state.config.push_gateway_delivery_url.as_ref() else { return; }; diff --git a/crates/buzz-relay/src/tunnel/directory.rs b/crates/buzz-relay/src/tunnel/directory.rs index 7cd6e22a2d..82542f1c87 100644 --- a/crates/buzz-relay/src/tunnel/directory.rs +++ b/crates/buzz-relay/src/tunnel/directory.rs @@ -87,6 +87,7 @@ return {current, known_generation} pub struct SessionDirectory { pool: deadpool_redis::Pool, lease_ttl: Duration, + db: Option, } /// Active session ownership lease read from Redis. @@ -179,6 +180,9 @@ pub enum DirectoryError { /// Lease TTL cannot be represented in Redis milliseconds. #[error("lease ttl must be at least 1ms and fit in i64 milliseconds")] InvalidLeaseTtl, + /// Durable community deletion fence rejected a serving write. + #[error("community write fenced: {0}")] + CommunityWriteFenced(String), } impl SessionDirectory { @@ -187,9 +191,37 @@ impl SessionDirectory { Self::with_lease_ttl(pool, DEFAULT_LEASE_TTL) } + /// Create a serving directory whose Redis mutations consult the durable + /// community write fence. + pub fn with_db(pool: deadpool_redis::Pool, db: buzz_db::Db) -> Self { + Self { + pool, + lease_ttl: DEFAULT_LEASE_TTL, + db: Some(db), + } + } + /// Create a directory backed by `pool` with an explicit lease TTL. pub fn with_lease_ttl(pool: deadpool_redis::Pool, lease_ttl: Duration) -> Self { - Self { pool, lease_ttl } + Self { + pool, + lease_ttl, + db: None, + } + } + + async fn begin_serving_write( + &self, + community_id: CommunityId, + ) -> Result, DirectoryError> { + match &self.db { + Some(db) => buzz_deletion::store(db) + .begin_serving_write(community_id) + .await + .map(Some) + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string())), + None => Ok(None), + } } /// Attempt to create/take over the session lease. @@ -204,6 +236,7 @@ impl SessionDirectory { owner_runtime_id: RuntimeId, profile: Profile, ) -> Result { + let _serving_write = self.begin_serving_write(community_id).await?; let keys = SessionKeys::new(community_id, session_id); let ttl_ms = ttl_ms(self.lease_ttl)?; let mut conn = self.pool.get().await?; @@ -244,6 +277,7 @@ impl SessionDirectory { /// Renew a lease only if the current Redis value exactly matches the /// caller's owner runtime and generation. pub async fn renew(&self, lease: &SessionLease) -> Result { + let _serving_write = self.begin_serving_write(lease.community_id).await?; let keys = SessionKeys::new(lease.community_id, lease.session_id); let ttl_ms = ttl_ms(self.lease_ttl)?; let mut conn = self.pool.get().await?; diff --git a/crates/buzz-relay/src/workflow_sink.rs b/crates/buzz-relay/src/workflow_sink.rs index 97c31c2561..ad474fafa7 100644 --- a/crates/buzz-relay/src/workflow_sink.rs +++ b/crates/buzz-relay/src/workflow_sink.rs @@ -334,6 +334,11 @@ impl ActionSink for RelayActionSink { broadcast: false, }); + buzz_deletion::store(&state.db) + .assert_serving_write_allowed(tenant.community()) + .await + .map_err(|error| ActionSinkError::Database(error.to_string()))?; + let (stored_event, was_inserted) = state .db .insert_event_with_thread_metadata( diff --git a/crates/buzz-workflow/Cargo.toml b/crates/buzz-workflow/Cargo.toml index d4813e56d4..7d361b1477 100644 --- a/crates/buzz-workflow/Cargo.toml +++ b/crates/buzz-workflow/Cargo.toml @@ -10,6 +10,7 @@ description = "YAML-as-code workflow engine for Buzz" [dependencies] buzz-core = { workspace = true } buzz-db = { workspace = true } +buzz-deletion = { workspace = true } hex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/buzz-workflow/src/executor.rs b/crates/buzz-workflow/src/executor.rs index e30541377e..8a52ae67ae 100644 --- a/crates/buzz-workflow/src/executor.rs +++ b/crates/buzz-workflow/src/executor.rs @@ -526,6 +526,19 @@ pub async fn dispatch_action( ) -> Result { use ActionDef::*; + // The workflow engine can outlive the serving request that spawned it. + // Revalidate the durable community fence immediately before every external + // side effect (message publish, webhook, delay/resume). A storage failure is + // a denial, never permission to continue. + let _serving_write = buzz_deletion::store(&engine.db) + .begin_serving_write(community_id) + .await + .map_err(|error| { + WorkflowError::WebhookError(format!( + "community write fence rejected workflow side effect: {error}" + )) + })?; + match action { SendMessage { text, channel } => { // Look up workflow metadata for destination validation and diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index b2778df28b..c04bb47ca2 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -145,6 +145,37 @@ volumes, and each init container must define an appropriate security context and resources. Empty `relay.command` and `relay.args` arrays preserve the image defaults; non-empty values override its entrypoint and arguments respectively. +## Whole-community deletion worker + +Whole-community logical deletion is CLI-only. The default chart remains inert: +`deletionWorker.enabled=false` renders no deletion pod, Service, Ingress, or +route, and the image ENTRYPOINT remains `buzz-relay`. + +Manual OSS operators run the durable engine from the existing image context: + +```sh +buzz-admin deletions submit --host community.example --requested-by operator +buzz-admin deletions approve --approved-by operator +buzz-admin deletions run # one job +buzz-admin deletions drain # current runnable queue +``` + +Hosted operators can opt into a dedicated no-ingress worker: + +```yaml +deletionWorker: + enabled: true + serviceAccountName: buzz-deletion-worker + existingSecret: buzz-deletion-worker-credentials +``` + +The dedicated Secret and service account should grant only the PostgreSQL, +Redis, and S3 delete/list capabilities needed by the worker. Its command is +`buzz-admin deletions worker`; the private `/_liveness` and `/_readiness` +listener exists only for pod probes, and SIGTERM marks readiness draining, +finishes/releases the current durable unit, then exits inside the configured +90-second window. Do not route public traffic to this pod. + ## Device pairing relay The chart can run Buzz's stateless pairing WebSocket relay as an independent diff --git a/deploy/charts/buzz/templates/deletion-worker-deployment.yaml b/deploy/charts/buzz/templates/deletion-worker-deployment.yaml new file mode 100644 index 0000000000..2e32ac8ec1 --- /dev/null +++ b/deploy/charts/buzz/templates/deletion-worker-deployment.yaml @@ -0,0 +1,95 @@ +{{- if .Values.deletionWorker.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "buzz.fullname" . }}-deletion-worker + labels: + {{- include "buzz.labels" . | nindent 4 }} + app.kubernetes.io/component: deletion-worker +spec: + replicas: {{ .Values.deletionWorker.replicaCount }} + strategy: + type: Recreate + selector: + matchLabels: + {{- include "buzz.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: deletion-worker + template: + metadata: + labels: + {{- include "buzz.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: deletion-worker + {{- with .Values.deletionWorker.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + # No Service/Ingress is created: only private pod health probes listen. + serviceAccountName: {{ required "deletionWorker.serviceAccountName is required when deletionWorker.enabled=true; use a dedicated identity" .Values.deletionWorker.serviceAccountName }} + automountServiceAccountToken: {{ .Values.deletionWorker.automountServiceAccountToken }} + terminationGracePeriodSeconds: {{ .Values.deletionWorker.terminationGracePeriodSeconds }} + securityContext: + {{- toYaml .Values.relay.securityContext | nindent 8 }} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: deletion-worker + image: {{ include "buzz.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + {{- toYaml .Values.relay.containerSecurityContext | nindent 12 }} + command: ["/usr/local/bin/buzz-admin"] + args: ["deletions", "worker"] + ports: + - { name: health, containerPort: {{ .Values.deletionWorker.healthPort }}, protocol: TCP } + env: + - { name: RELAY_URL, value: {{ .Values.relayUrl | quote }} } + - { name: BUZZ_DELETION_HEALTH_ADDR, value: {{ printf "0.0.0.0:%v" .Values.deletionWorker.healthPort | quote }} } + - { name: BUZZ_DELETION_STORAGE_MAX_OBJECTS, value: {{ .Values.deletionWorker.storageMaxObjects | quote }} } + {{- $s3Endpoint := include "buzz.s3Endpoint" . }} + {{- if $s3Endpoint }} + - { name: BUZZ_S3_ENDPOINT, value: {{ $s3Endpoint | quote }} } + {{- end }} + - { name: BUZZ_S3_BUCKET, value: {{ .Values.s3.bucket | quote }} } + {{- if .Values.s3.region }} + - { name: BUZZ_S3_REGION, value: {{ .Values.s3.region | quote }} } + {{- end }} + - { name: BUZZ_S3_ADDRESSING_STYLE, value: {{ .Values.s3.addressingStyle | quote }} } + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ required "deletionWorker.existingSecret is required when deletionWorker.enabled=true" .Values.deletionWorker.existingSecret }} + key: DATABASE_URL + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: {{ .Values.deletionWorker.existingSecret }} + key: REDIS_URL + - name: BUZZ_S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.deletionWorker.existingSecret }} + key: BUZZ_S3_ACCESS_KEY + optional: true + - name: BUZZ_S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ .Values.deletionWorker.existingSecret }} + key: BUZZ_S3_SECRET_KEY + optional: true + {{- with .Values.deletionWorker.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + livenessProbe: + httpGet: { path: /_liveness, port: health } + periodSeconds: 10 + timeoutSeconds: 3 + readinessProbe: + httpGet: { path: /_readiness, port: health } + periodSeconds: 5 + timeoutSeconds: 3 + resources: + {{- toYaml .Values.deletionWorker.resources | nindent 12 }} +{{- end }} diff --git a/deploy/charts/buzz/tests/deletion_worker_test.yaml b/deploy/charts/buzz/tests/deletion_worker_test.yaml new file mode 100644 index 0000000000..e76604954c --- /dev/null +++ b/deploy/charts/buzz/tests/deletion_worker_test.yaml @@ -0,0 +1,45 @@ +suite: deletion worker render +templates: + - templates/deletion-worker-deployment.yaml +tests: + - it: remains completely inert by default + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: http://minio:9000 + s3.accessKey: a + s3.secretKey: s + asserts: + - hasDocuments: + count: 0 + + - it: renders a dedicated no-ingress worker identity only when enabled + set: + relayUrl: wss://buzz.example.com + ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" + externalPostgresql.url: postgres://u:p@h:5432/d + externalRedis.url: redis://h:6379 + s3.endpoint: http://minio:9000 + s3.accessKey: a + s3.secretKey: s + deletionWorker.enabled: true + deletionWorker.serviceAccountName: buzz-deletion-worker + deletionWorker.existingSecret: buzz-deletion-worker-credentials + asserts: + - equal: + path: kind + value: Deployment + - equal: + path: spec.template.spec.serviceAccountName + value: buzz-deletion-worker + - equal: + path: spec.template.spec.containers[0].command[0] + value: /usr/local/bin/buzz-admin + - equal: + path: spec.template.spec.containers[0].args + value: [deletions, worker] + - equal: + path: spec.template.spec.containers[0].readinessProbe.httpGet.path + value: /_readiness diff --git a/deploy/charts/buzz/tests/render_test.yaml b/deploy/charts/buzz/tests/render_test.yaml index cf08210781..805d40df87 100644 --- a/deploy/charts/buzz/tests/render_test.yaml +++ b/deploy/charts/buzz/tests/render_test.yaml @@ -3,6 +3,7 @@ suite: production render # (e.g. checksum/secret include of secret-chart.yaml) resolve at render time. templates: - templates/deployment.yaml + - templates/deletion-worker-deployment.yaml - templates/secret-chart.yaml - templates/serviceaccount.yaml - templates/service.yaml diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index 42b09f1b3e..72ddb55072 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -397,3 +397,26 @@ serviceMonitor: # ── Free-form extra manifests ──────────────────────────────────────────────── extraManifests: [] + +# ── Whole-community deletion worker ────────────────────────────────────────── +# Strictly inert by default. Hosted operators may enable this dedicated, +# no-ingress identity; manual OSS operation uses `buzz-admin deletions run|drain` +# from the existing image instead. +deletionWorker: + enabled: false + replicaCount: 1 + serviceAccountName: "" + automountServiceAccountToken: false + existingSecret: "" + healthPort: 8080 + terminationGracePeriodSeconds: 90 + storageMaxObjects: 1000000 + podAnnotations: {} + extraEnv: [] + resources: + requests: + cpu: "250m" + memory: "256Mi" + limits: + cpu: "1" + memory: "1Gi" diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql new file mode 100644 index 0000000000..c2e241f258 --- /dev/null +++ b/migrations/0028_community_deletion.sql @@ -0,0 +1,239 @@ +-- Durable, CLI-only whole-community deletion control plane. +-- +-- The community row is never removed: it becomes the permanent name tombstone. +-- All destructive progress is lease/fence/checkpoint guarded and every existing +-- community-scoped table receives the same database-enforced write fence. +ALTER TABLE communities + ADD COLUMN deletion_state TEXT NOT NULL DEFAULT 'active' + CHECK (deletion_state IN ('active', 'fenced', 'tombstone')), + ADD COLUMN deletion_fence_generation BIGINT NOT NULL DEFAULT 0 + CHECK (deletion_fence_generation >= 0), + ADD COLUMN deleted_at TIMESTAMPTZ; + +CREATE TABLE community_deletion_requests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL UNIQUE REFERENCES communities(id), + community_host TEXT NOT NULL, + stage TEXT NOT NULL DEFAULT 'submitted' CHECK (stage IN ( + 'submitted', 'inventoried', 'approved', 'fenced', 'drained', + 'bindings_removed', 'postgres_purged', 'cache_purged', + 'logically_verified', 'retention_pending' + )), + requested_by TEXT NOT NULL, + reason TEXT, + schema_manifest JSONB, + storage_manifest JSONB, + inventory_manifest JSONB, + inventory_digest BYTEA CHECK (inventory_digest IS NULL OR length(inventory_digest) = 32), + inventory_frozen_at TIMESTAMPTZ, + fence_generation BIGINT CHECK (fence_generation IS NULL OR fence_generation > 0), + lease_owner TEXT, + lease_generation BIGINT NOT NULL DEFAULT 0 CHECK (lease_generation >= 0), + lease_until TIMESTAMPTZ, + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error TEXT, + last_error_at TIMESTAMPTZ, + blocked_at TIMESTAMPTZ, + blocked_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + CHECK ((blocked_at IS NULL) = (blocked_reason IS NULL)), + CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)) +); +CREATE INDEX community_deletion_requests_runnable + ON community_deletion_requests (next_attempt_at, created_at) + WHERE blocked_at IS NULL + AND stage IN ('approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified'); +CREATE INDEX community_deletion_requests_lease + ON community_deletion_requests (lease_until) + WHERE lease_owner IS NOT NULL; + +CREATE TABLE community_deletion_approvals ( + request_id UUID PRIMARY KEY REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), + approved_by TEXT NOT NULL, + note TEXT, + approved_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE community_deletion_checkpoints ( + request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + sequence BIGINT GENERATED ALWAYS AS IDENTITY, + stage TEXT NOT NULL, + unit_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('started', 'completed', 'failed')), + lease_generation BIGINT NOT NULL CHECK (lease_generation > 0), + attempts INTEGER NOT NULL DEFAULT 1 CHECK (attempts > 0), + detail JSONB NOT NULL DEFAULT '{}'::jsonb, + error TEXT, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + PRIMARY KEY (request_id, sequence), + UNIQUE (request_id, stage, unit_key), + CHECK ((status = 'completed') = (completed_at IS NOT NULL)), + CHECK ((status = 'failed') = (error IS NOT NULL)) +); + +CREATE TABLE community_deletion_retention_exceptions ( + request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + exception_key TEXT NOT NULL, + reason TEXT NOT NULL, + expires_at TIMESTAMPTZ, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (request_id, exception_key) +); + +CREATE TABLE community_deletion_executor_heartbeats ( + executor_id TEXT PRIMARY KEY, + mode TEXT NOT NULL CHECK (mode IN ('run', 'drain', 'worker')), + request_id UUID REFERENCES community_deletion_requests(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + draining BOOLEAN NOT NULL DEFAULT false, + stopped_at TIMESTAMPTZ +); + +INSERT INTO _operator_global_tables (table_name, reason) VALUES + ('community_deletion_requests', 'deployment deletion lifecycle and frozen inventory'), + ('community_deletion_approvals', 'deployment operator destructive approvals'), + ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), + ('community_deletion_retention_exceptions', 'deployment retention holds and expiry exceptions'), + ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); + +-- Shared lock key used by both the trigger and the deletion engine. Every +-- ordinary tenant mutation takes the shared xact lock before checking state; +-- the fence transition takes the exclusive xact lock, so an already-open write +-- transaction cannot commit behind the fence and no new writer can slip ahead. +CREATE FUNCTION community_deletion_lock_key(target UUID) RETURNS BIGINT +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT hashtextextended('buzz-community-deletion:' || target::text, 0) +$$; + +CREATE FUNCTION enforce_community_write_fence() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +DECLARE + target UUID; + lifecycle TEXT; + generation BIGINT; + executor_community TEXT; + executor_generation TEXT; +BEGIN + IF TG_OP = 'INSERT' THEN + target := NEW.community_id; + ELSE + target := OLD.community_id; + END IF; + + -- Nullable operator-attribution rows without a tenant are unrelated. + IF target IS NULL THEN + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; + END IF; + + PERFORM pg_advisory_xact_lock_shared(community_deletion_lock_key(target)); + SELECT deletion_state, deletion_fence_generation + INTO lifecycle, generation + FROM communities + WHERE id = target; + IF NOT FOUND THEN + RAISE EXCEPTION 'community write rejected: community % is missing', target + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + + executor_community := current_setting('buzz.deletion_executor_community', true); + executor_generation := current_setting('buzz.deletion_fence_generation', true); + IF executor_community = target::text + AND executor_generation ~ '^[0-9]+$' + AND executor_generation::BIGINT = generation THEN + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; + END IF; + + IF lifecycle <> 'active' THEN + RAISE EXCEPTION 'community write fenced: community % generation %', target, generation + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; +END +$$; + +-- Protect the tombstone row itself. Normal updates are permitted only while the +-- row is active and do not change deletion metadata. Deletion executor updates +-- must present the exact durable generation in session-local GUCs. +CREATE FUNCTION enforce_community_tombstone() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +DECLARE + executor_community TEXT := current_setting('buzz.deletion_executor_community', true); + executor_generation TEXT := current_setting('buzz.deletion_fence_generation', true); + expected_generation BIGINT; +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'community tombstones are permanent' + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + + expected_generation := CASE + WHEN NEW.deletion_fence_generation > OLD.deletion_fence_generation + THEN NEW.deletion_fence_generation + ELSE OLD.deletion_fence_generation + END; + IF executor_community = OLD.id::text + AND executor_generation ~ '^[0-9]+$' + AND executor_generation::BIGINT = expected_generation THEN + RETURN NEW; + END IF; + + IF OLD.deletion_state <> 'active' + OR NEW.deletion_state <> OLD.deletion_state + OR NEW.deletion_fence_generation <> OLD.deletion_fence_generation + OR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THEN + RAISE EXCEPTION 'community tombstone mutation rejected: community % generation %', + OLD.id, OLD.deletion_fence_generation + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN NEW; +END +$$; + +CREATE TRIGGER communities_deletion_tombstone +BEFORE UPDATE OR DELETE ON communities +FOR EACH ROW EXECUTE FUNCTION enforce_community_tombstone(); + +-- Attach the universal fence to every existing table carrying community_id, +-- including deployment-private sidecars whose community_id is provenance. +-- Deletion control-plane tables are excluded: they must remain writable while +-- the target tenant is fenced. +DO $$ +DECLARE + table_name TEXT; +BEGIN + FOR table_name IN + SELECT c.relname + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition + AND a.attname = 'community_id' + AND NOT a.attisdropped + AND c.relname NOT IN ( + 'community_deletion_requests', + 'community_deletion_approvals', + 'community_deletion_checkpoints', + 'community_deletion_retention_exceptions', + 'community_deletion_executor_heartbeats' + ) + LOOP + EXECUTE format( + 'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %I ' + 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', + 'community_write_fence_' || table_name, + table_name + ); + END LOOP; +END +$$; diff --git a/schema/schema.sql b/schema/schema.sql index 4dac29176d..1db7209b10 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -59,6 +59,9 @@ CREATE TABLE communities ( icon TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), archived_at TIMESTAMPTZ, + deletion_state TEXT NOT NULL DEFAULT 'active' CHECK (deletion_state IN ('active', 'fenced', 'tombstone')), + deletion_fence_generation BIGINT NOT NULL DEFAULT 0 CHECK (deletion_fence_generation >= 0), + deleted_at TIMESTAMPTZ, CONSTRAINT chk_communities_id_not_nil CHECK (id <> '00000000-0000-0000-0000-000000000000'::uuid) ); @@ -1080,3 +1083,176 @@ INSERT INTO replica_heartbeat (id) VALUES (1); INSERT INTO _operator_global_tables (table_name, reason) VALUES ('replica_heartbeat', 'single-row replication freshness token; describes deployment topology, never tenant data'); + +-- ── Whole-community deletion control plane (migration 0028) ───────────────── +CREATE TABLE community_deletion_requests ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL UNIQUE REFERENCES communities(id), + community_host TEXT NOT NULL, + stage TEXT NOT NULL DEFAULT 'submitted' CHECK (stage IN ( + 'submitted', 'inventoried', 'approved', 'fenced', 'drained', + 'bindings_removed', 'postgres_purged', 'cache_purged', + 'logically_verified', 'retention_pending' + )), + requested_by TEXT NOT NULL, + reason TEXT, + schema_manifest JSONB, + storage_manifest JSONB, + inventory_manifest JSONB, + inventory_digest BYTEA CHECK (inventory_digest IS NULL OR length(inventory_digest) = 32), + inventory_frozen_at TIMESTAMPTZ, + fence_generation BIGINT CHECK (fence_generation IS NULL OR fence_generation > 0), + lease_owner TEXT, + lease_generation BIGINT NOT NULL DEFAULT 0 CHECK (lease_generation >= 0), + lease_until TIMESTAMPTZ, + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_error TEXT, + last_error_at TIMESTAMPTZ, + blocked_at TIMESTAMPTZ, + blocked_reason TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + CHECK ((blocked_at IS NULL) = (blocked_reason IS NULL)), + CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)) +); +CREATE INDEX community_deletion_requests_runnable + ON community_deletion_requests (next_attempt_at, created_at) + WHERE blocked_at IS NULL + AND stage IN ('approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified'); +CREATE INDEX community_deletion_requests_lease + ON community_deletion_requests (lease_until) WHERE lease_owner IS NOT NULL; + +CREATE TABLE community_deletion_approvals ( + request_id UUID PRIMARY KEY REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), + approved_by TEXT NOT NULL, + note TEXT, + approved_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE TABLE community_deletion_checkpoints ( + request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + sequence BIGINT GENERATED ALWAYS AS IDENTITY, + stage TEXT NOT NULL, + unit_key TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ('started', 'completed', 'failed')), + lease_generation BIGINT NOT NULL CHECK (lease_generation > 0), + attempts INTEGER NOT NULL DEFAULT 1 CHECK (attempts > 0), + detail JSONB NOT NULL DEFAULT '{}'::jsonb, + error TEXT, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + PRIMARY KEY (request_id, sequence), + UNIQUE (request_id, stage, unit_key), + CHECK ((status = 'completed') = (completed_at IS NOT NULL)), + CHECK ((status = 'failed') = (error IS NOT NULL)) +); +CREATE TABLE community_deletion_retention_exceptions ( + request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + exception_key TEXT NOT NULL, + reason TEXT NOT NULL, + expires_at TIMESTAMPTZ, + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (request_id, exception_key) +); +CREATE TABLE community_deletion_executor_heartbeats ( + executor_id TEXT PRIMARY KEY, + mode TEXT NOT NULL CHECK (mode IN ('run', 'drain', 'worker')), + request_id UUID REFERENCES community_deletion_requests(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + draining BOOLEAN NOT NULL DEFAULT false, + stopped_at TIMESTAMPTZ +); +INSERT INTO _operator_global_tables (table_name, reason) VALUES + ('community_deletion_requests', 'deployment deletion lifecycle and frozen inventory'), + ('community_deletion_approvals', 'deployment operator destructive approvals'), + ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), + ('community_deletion_retention_exceptions', 'deployment retention holds and expiry exceptions'), + ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); + +CREATE FUNCTION community_deletion_lock_key(target UUID) RETURNS BIGINT +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT hashtextextended('buzz-community-deletion:' || target::text, 0) +$$; +CREATE FUNCTION enforce_community_write_fence() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +DECLARE + target UUID; + lifecycle TEXT; + generation BIGINT; + executor_community TEXT; + executor_generation TEXT; +BEGIN + IF TG_OP = 'INSERT' THEN target := NEW.community_id; ELSE target := OLD.community_id; END IF; + IF target IS NULL THEN RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; END IF; + PERFORM pg_advisory_xact_lock_shared(community_deletion_lock_key(target)); + SELECT deletion_state, deletion_fence_generation INTO lifecycle, generation + FROM communities WHERE id = target; + IF NOT FOUND THEN + RAISE EXCEPTION 'community write rejected: community % is missing', target + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + executor_community := current_setting('buzz.deletion_executor_community', true); + executor_generation := current_setting('buzz.deletion_fence_generation', true); + IF executor_community = target::text AND executor_generation ~ '^[0-9]+$' + AND executor_generation::BIGINT = generation THEN + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; + END IF; + IF lifecycle <> 'active' THEN + RAISE EXCEPTION 'community write fenced: community % generation %', target, generation + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; +END +$$; +CREATE FUNCTION enforce_community_tombstone() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +DECLARE + executor_community TEXT := current_setting('buzz.deletion_executor_community', true); + executor_generation TEXT := current_setting('buzz.deletion_fence_generation', true); + expected_generation BIGINT; +BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'community tombstones are permanent' + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + expected_generation := CASE WHEN NEW.deletion_fence_generation > OLD.deletion_fence_generation + THEN NEW.deletion_fence_generation ELSE OLD.deletion_fence_generation END; + IF executor_community = OLD.id::text AND executor_generation ~ '^[0-9]+$' + AND executor_generation::BIGINT = expected_generation THEN RETURN NEW; END IF; + IF OLD.deletion_state <> 'active' OR NEW.deletion_state <> OLD.deletion_state + OR NEW.deletion_fence_generation <> OLD.deletion_fence_generation + OR NEW.deleted_at IS DISTINCT FROM OLD.deleted_at THEN + RAISE EXCEPTION 'community tombstone mutation rejected: community % generation %', + OLD.id, OLD.deletion_fence_generation + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN NEW; +END +$$; +CREATE TRIGGER communities_deletion_tombstone BEFORE UPDATE OR DELETE ON communities +FOR EACH ROW EXECUTE FUNCTION enforce_community_tombstone(); +DO $$ +DECLARE table_name TEXT; +BEGIN + FOR table_name IN + SELECT c.relname FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p') AND NOT c.relispartition + AND a.attname = 'community_id' AND NOT a.attisdropped + AND c.relname NOT IN ('community_deletion_requests', 'community_deletion_approvals', + 'community_deletion_checkpoints', 'community_deletion_retention_exceptions', + 'community_deletion_executor_heartbeats') + LOOP + EXECUTE format('CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %I ' + 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', + 'community_write_fence_' || table_name, table_name); + END LOOP; +END +$$; From 08981b590f0465da941f83a47307dcfd374e69ac Mon Sep 17 00:00:00 2001 From: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Date: Fri, 31 Jul 2026 15:48:31 -0700 Subject: [PATCH 02/51] wip: checkpoint deletion remediation lanes Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> --- Cargo.lock | 2 + crates/buzz-db/src/deletion.rs | 362 ++++++++-- crates/buzz-db/src/error.rs | 5 + crates/buzz-db/src/lib.rs | 17 +- crates/buzz-db/src/migration.rs | 2 + crates/buzz-deletion/Cargo.toml | 4 + crates/buzz-deletion/src/lib.rs | 643 +++++++++++++++--- crates/buzz-media/src/storage.rs | 5 + crates/buzz-relay/src/api/git/transport.rs | 156 +++-- crates/buzz-relay/src/api/invites.rs | 4 +- crates/buzz-relay/src/api/media.rs | 156 +++-- crates/buzz-relay/src/api/mesh_demo.rs | 31 +- crates/buzz-relay/src/handlers/event.rs | 77 ++- crates/buzz-relay/src/handlers/ingest.rs | 7 - crates/buzz-relay/src/mesh_boot.rs | 21 +- crates/buzz-relay/src/push_runtime.rs | 28 +- crates/buzz-relay/src/tunnel/directory.rs | 83 ++- crates/buzz-relay/src/workflow_sink.rs | 5 - crates/buzz-workflow/src/executor.rs | 338 ++++----- .../templates/deletion-worker-deployment.yaml | 2 - .../buzz/tests/deletion_worker_test.yaml | 4 + deploy/local/build-and-deploy.sh | 24 +- migrations/0028_community_deletion.sql | 27 +- schema/schema.sql | 28 +- 24 files changed, 1483 insertions(+), 548 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1420c13305..62a7cef6be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1034,6 +1034,8 @@ dependencies = [ "redis", "serde", "serde_json", + "sqlx", + "thiserror 2.0.18", "tokio", "tokio-util", "uuid", diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index a8edb3a169..d9b680c2cf 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -38,6 +38,7 @@ pub const CONTROL_PLANE_TABLES: &[&str] = &[ "community_deletion_executor_heartbeats", "community_deletion_requests", "community_deletion_retention_exceptions", + "community_serving_write_leases", ]; /// Expected community-scoped tables purged by V1. @@ -203,7 +204,7 @@ impl FromStr for DeletionStage { "cache_purged" => Ok(Self::CachePurged), "logically_verified" => Ok(Self::LogicallyVerified), "retention_pending" => Ok(Self::RetentionPending), - other => Err(DbError::InvalidData(format!( + other => Err(DbError::DeletionSafety(format!( "unknown community deletion stage: {other}" ))), } @@ -228,8 +229,10 @@ pub struct DeletionRequest { pub reason: Option, /// Frozen catalog manifest. pub schema_manifest: Option, - /// Frozen storage manifest. + /// Frozen storage taxonomy manifest observed at submission. pub storage_manifest: Option, + /// Destructive storage manifest frozen after the durable fence. + pub destructive_storage_manifest: Option, /// Frozen inventory aggregate. pub inventory_manifest: Option, /// Hex SHA-256 of the frozen inventory. @@ -399,13 +402,23 @@ pub struct ClaimedDeletion { pub lease: LeaseToken, } -/// Held shared advisory lock proving a serving write began before fencing. -/// -/// Dropping the guard rolls back its read-only transaction and releases the -/// lock. The destructive fence takes the matching exclusive lock, so it cannot -/// advance while an S3/Redis/webhook side effect still holds this guard. -pub struct ServingWriteGuard { - _tx: Transaction<'static, Postgres>, +/// Short-lived durable lease for an external serving side effect. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ServingWriteLease { + /// Lease row identifier. + pub id: Uuid, + /// Community protected by this lease. + pub community_id: CommunityId, + /// Operation category for diagnostics. + pub operation: String, + /// Process/executor identity. + pub owner: String, + /// Monotonic lease generation. + pub generation: i64, + /// Community fence generation observed when the lease was acquired. + pub fence_generation: i64, + /// Lease expiry. + pub lease_until: DateTime, } /// PostgreSQL deletion adapter. Clone is cheap. @@ -420,6 +433,17 @@ impl DeletionStore { Self { pool } } + /// Check deletion control-plane/schema connectivity. + pub async fn ping(&self) -> bool { + sqlx::query_scalar::<_, i32>( + "SELECT 1 FROM _sqlx_migrations WHERE version = $1 AND success LIMIT 1", + ) + .bind(EXPECTED_MIGRATION_VERSION) + .fetch_optional(&self.pool) + .await + .is_ok_and(|row| row == Some(1)) + } + /// Persist a request. Only active non-tombstone communities may be submitted. pub async fn submit( &self, @@ -460,7 +484,7 @@ impl DeletionStore { .await?; match row { Some(row) => row_to_request(row), - None => Err(DbError::InvalidData(format!( + None => Err(DbError::DeletionSafety(format!( "community {community_host:?} is missing, already requested, fenced, or tombstoned" ))), } @@ -565,7 +589,7 @@ impl DeletionStore { .fetch_one(&self.pool) .await?; if migration_version != EXPECTED_MIGRATION_VERSION { - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "community deletion schema migration drift: expected {EXPECTED_MIGRATION_VERSION}, got {migration_version}" ))); } @@ -585,7 +609,7 @@ impl DeletionStore { .difference(&expected) .cloned() .collect::>(); - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "community deletion catalog drift (missing={}, unknown={})", missing.join(","), unknown.join(",") @@ -598,26 +622,18 @@ impl DeletionStore { .difference(&fenced_tables) .cloned() .collect::>(); - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "community deletion write-fence drift (missing={})", missing.join(",") ))); } - let mut row_counts = BTreeMap::new(); - for table in &live_tables { - let sql = format!("SELECT count(*) FROM {table} WHERE community_id = $1"); - let count: i64 = sqlx::query_scalar(AssertSqlSafe(sql)) - .bind(community.as_uuid()) - .fetch_one(&self.pool) - .await?; - row_counts.insert(table.clone(), count); - } + let _ = community; // counts are intentionally not approval-bound for a live tenant. Ok(SchemaManifest { revision: CATALOG_REVISION, migration_version, scoped_tables: live_tables.into_iter().collect(), - row_counts, + row_counts: BTreeMap::new(), fenced_tables: fenced_tables.into_iter().collect(), }) } @@ -652,7 +668,7 @@ impl DeletionStore { .fetch_optional(&self.pool) .await? .ok_or_else(|| { - DbError::InvalidData(format!( + DbError::DeletionSafety(format!( "deletion {request_id} is not an unblocked submitted request" )) })?; @@ -675,7 +691,7 @@ impl DeletionStore { .fetch_optional(&mut *tx) .await? .ok_or_else(|| { - DbError::InvalidData(format!( + DbError::DeletionSafety(format!( "deletion {request_id} is not an unblocked inventoried request" )) })?; @@ -772,6 +788,22 @@ impl DeletionStore { .transpose() } + /// Verify that a deletion lease/fence token is still current for a stage. + pub async fn verify_execution_token( + &self, + token: &LeaseToken, + stage: DeletionStage, + ) -> Result<()> { + let mut tx = self.pool.begin().await?; + if let Some(generation) = token.fence_generation { + verify_lease_and_fence(&mut tx, token, stage, generation).await?; + } else { + verify_lease(&mut tx, token, stage).await?; + } + tx.commit().await?; + Ok(()) + } + /// Renew an owned claim and persist executor liveness. pub async fn heartbeat( &self, @@ -857,10 +889,10 @@ impl DeletionStore { .fetch_one(&mut *tx) .await?; let generation = current_generation.checked_add(1).ok_or_else(|| { - DbError::InvalidData("community deletion fence generation overflow".to_string()) + DbError::DeletionSafety("community deletion fence generation overflow".to_string()) })?; set_executor_gucs(&mut tx, token.community_id, generation).await?; - sqlx::query( + let affected = sqlx::query( "UPDATE communities SET deletion_state = 'fenced', \ deletion_fence_generation = $2, archived_at = COALESCE(archived_at, now()) \ WHERE id = $1 AND deletion_state = 'active'", @@ -868,7 +900,14 @@ impl DeletionStore { .bind(token.community_id.as_uuid()) .bind(generation) .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} is no longer active while fencing", + token.community_id + ))); + } advance_request_tx( &mut tx, token, @@ -889,11 +928,72 @@ impl DeletionStore { Ok(generation) } + /// Freeze the exact post-fence storage binding manifest. + pub async fn freeze_destructive_storage_manifest( + &self, + token: &LeaseToken, + manifest: &StorageManifest, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + validate_storage_manifest(manifest)?; + let affected = sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_manifest = COALESCE(destructive_storage_manifest, $4), \ + destructive_storage_frozen_at = COALESCE(destructive_storage_frozen_at, now()), \ + updated_at = now() \ + WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3 \ + AND stage = 'fenced' \ + AND (destructive_storage_manifest IS NULL \ + OR destructive_storage_manifest = $4) \ + RETURNING id", + ) + .bind(token.request_id) + .bind(&token.owner) + .bind(token.generation) + .bind(serde_json::to_value(manifest)?) + .fetch_optional(&mut *tx) + .await?; + if affected.is_none() { + return Err(DbError::DeletionSafety(format!( + "destructive storage manifest changed or deletion lease is stale for request {}", + token.request_id + ))); + } + tx.commit().await?; + Ok(()) + } + + /// Return whether all pre-fence external side-effect leases have expired or released. + pub async fn serving_writes_drained(&self, community: CommunityId) -> Result { + sqlx::query_scalar( + "SELECT NOT EXISTS(SELECT 1 FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now())", + ) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + /// Verify fence ownership and record that serving writes drained. pub async fn mark_drained(&self, token: &LeaseToken) -> Result<()> { let generation = require_fence_generation(token)?; let mut tx = self.pool.begin().await?; verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + let active_serving_writes: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now())", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + if active_serving_writes { + return Err(DbError::DeletionSafety( + "serving writes have not drained".to_string(), + )); + } advance_request_tx( &mut tx, token, @@ -1031,7 +1131,7 @@ impl DeletionStore { .fetch_one(&mut *tx) .await?; if !tombstone { - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "community {} tombstone/fence verification failed", token.community_id ))); @@ -1044,7 +1144,7 @@ impl DeletionStore { .fetch_one(&mut *tx) .await?; if remains { - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "logical verification found tenant rows in {table}" ))); } @@ -1196,37 +1296,141 @@ impl DeletionStore { } } - /// Begin a serving write and hold the community's shared deletion lock. - pub async fn begin_serving_write(&self, community: CommunityId) -> Result { + /// Acquire a durable, expiring lease for an external serving side effect. + /// + /// The short transaction shares the same advisory lock as the destructive + /// fence. The fence therefore orders after all acquisitions that began + /// first, changes lifecycle state, then refuses every later acquisition. + pub async fn acquire_serving_write_lease( + &self, + community: CommunityId, + operation: &str, + owner: &str, + lease_duration: Duration, + ) -> Result { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); let mut tx = self.pool.begin().await?; sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") .bind(community.as_uuid()) .execute(&mut *tx) .await?; - let state: Option = sqlx::query_scalar( - "SELECT deletion_state FROM communities WHERE id = $1 AND deleted_at IS NULL", + let row = sqlx::query( + "INSERT INTO community_serving_write_leases \ + (community_id, operation, owner, fence_generation, lease_until) \ + SELECT id, $2, $3, deletion_fence_generation, \ + now() + make_interval(secs => $4) \ + FROM communities WHERE id = $1 AND deletion_state = 'active' \ + AND deleted_at IS NULL \ + RETURNING id, generation, fence_generation, lease_until", ) .bind(community.as_uuid()) + .bind(operation) + .bind(owner) + .bind(lease_seconds) .fetch_optional(&mut *tx) - .await?; - match state.as_deref() { - Some("active") => Ok(ServingWriteGuard { _tx: tx }), - Some(other) => Err(DbError::AccessDenied(format!( - "community {community} is write-fenced ({other})" - ))), - None => Err(DbError::AccessDenied(format!( - "community {community} is missing or tombstoned" - ))), - } + .await? + .ok_or_else(|| { + DbError::AccessDenied(format!("community {community} is write-fenced or missing")) + })?; + let lease = ServingWriteLease { + id: row.try_get("id")?, + community_id: community, + operation: operation.to_owned(), + owner: owner.to_owned(), + generation: row.try_get("generation")?, + fence_generation: row.try_get("fence_generation")?, + lease_until: row.try_get("lease_until")?, + }; + tx.commit().await?; + Ok(lease) } - /// Ensure the durable community write fence allows serving writes. - pub async fn assert_serving_write_allowed(&self, community: CommunityId) -> Result<()> { - let guard = self.begin_serving_write(community).await?; - drop(guard); + /// Renew an external side-effect lease only while ownership and the + /// community's active lifecycle are still current. + pub async fn renew_serving_write_lease( + &self, + lease: &mut ServingWriteLease, + lease_duration: Duration, + ) -> Result<()> { + let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let lease_until: Option> = sqlx::query_scalar( + "UPDATE community_serving_write_leases lease \ + SET lease_until = now() + make_interval(secs => $6), heartbeat_at = now() \ + FROM communities community \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() \ + AND community.id = lease.community_id \ + AND ((community.deletion_state = 'active' \ + AND community.deletion_fence_generation = lease.fence_generation) \ + OR (community.deletion_state = 'fenced' \ + AND community.deletion_fence_generation = lease.fence_generation + 1)) \ + RETURNING lease.lease_until", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .bind(lease_seconds) + .fetch_optional(&self.pool) + .await?; + lease.lease_until = lease_until.ok_or_else(|| { + DbError::AccessDenied(format!("stale serving write lease {}", lease.id)) + })?; Ok(()) } + /// Release a serving side-effect lease. A stale release is harmless. + pub async fn release_serving_write_lease(&self, lease: &ServingWriteLease) -> Result { + let deleted = sqlx::query( + "DELETE FROM community_serving_write_leases \ + WHERE id = $1 AND community_id = $2 AND owner = $3 AND generation = $4 \ + AND fence_generation = $5", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .execute(&self.pool) + .await? + .rows_affected(); + Ok(deleted == 1) + } + + /// Check that an external side-effect lease and the community's active + /// lifecycle are still current. + pub async fn verify_serving_write_lease(&self, lease: &ServingWriteLease) -> Result<()> { + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ + JOIN communities community ON community.id = lease.community_id \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() \ + AND community.deleted_at IS NULL \ + AND ((community.deletion_state = 'active' \ + AND community.deletion_fence_generation = lease.fence_generation) \ + OR (community.deletion_state = 'fenced' \ + AND community.deletion_fence_generation = lease.fence_generation + 1)))", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .fetch_one(&self.pool) + .await?; + if valid { + Ok(()) + } else { + Err(DbError::AccessDenied(format!( + "stale serving write lease {}", + lease.id + ))) + } + } + /// Whether a community remains active and serving-write eligible. pub async fn is_serving_active(&self, community: CommunityId) -> Result { sqlx::query_scalar( @@ -1273,6 +1477,7 @@ impl DeletionStore { 'community_deletion_approvals', 'community_deletion_checkpoints', 'community_deletion_retention_exceptions', + 'community_serving_write_leases', 'community_deletion_executor_heartbeats' ) ORDER BY c.relname @@ -1307,19 +1512,19 @@ impl DeletionStore { /// Fail closed when storage inventory reports unknown or unsupported data. pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { if manifest.version != 1 { - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "unsupported storage manifest version {}", manifest.version ))); } if !manifest.unknown_keys.is_empty() { - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "unknown object-store keys block deletion: {}", manifest.unknown_keys.join(",") ))); } if !manifest.unsupported_version_keys.is_empty() { - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "unsupported object versions block deletion: {}", manifest.unsupported_version_keys.join(",") ))); @@ -1406,7 +1611,7 @@ async fn advance_request_tx( fence_generation: Option, ) -> Result<()> { if from.next() != Some(to) { - return Err(DbError::InvalidData(format!( + return Err(DbError::DeletionSafety(format!( "illegal deletion transition {from} -> {to}" ))); } @@ -1511,6 +1716,7 @@ fn row_to_request(row: sqlx::postgres::PgRow) -> Result { reason: row.try_get("reason")?, schema_manifest: row.try_get("schema_manifest")?, storage_manifest: row.try_get("storage_manifest")?, + destructive_storage_manifest: row.try_get("destructive_storage_manifest")?, inventory_manifest: row.try_get("inventory_manifest")?, inventory_digest: digest.map(hex::encode), fence_generation: row.try_get("fence_generation")?, @@ -1536,7 +1742,7 @@ fn stale_lease_error(token: &LeaseToken) -> DbError { fn require_fence_generation(token: &LeaseToken) -> Result { token.fence_generation.ok_or_else(|| { - DbError::InvalidData(format!( + DbError::DeletionSafety(format!( "deletion {} has no durable fence generation", token.request_id )) @@ -1810,7 +2016,7 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn serving_guard_blocks_fence_until_external_side_effect_finishes() { + async fn external_serving_lease_blocks_drain_without_holding_a_pool_connection() { let (db, store) = store().await; let (request, _) = inventoried_request(&db, &store).await; store @@ -1822,27 +2028,47 @@ mod postgres_tests { .await .expect("claim") .expect("won claim"); - let guard = store - .begin_serving_write(request.community_id) + let mut serving = store + .acquire_serving_write_lease( + request.community_id, + "test_external", + "test-owner", + DEFAULT_LEASE_DURATION, + ) .await - .expect("serving write guard"); - let store_for_fence = store.clone(); - let lease = claim.lease.clone(); - let fencing = tokio::spawn(async move { store_for_fence.fence(&lease).await }); - tokio::time::sleep(Duration::from_millis(50)).await; + .expect("serving lease"); + let generation = store.fence(&claim.lease).await.expect("fence"); + store + .renew_serving_write_lease(&mut serving, DEFAULT_LEASE_DURATION) + .await + .expect("pre-fence side effect continues heartbeating while fenced"); + store + .verify_serving_write_lease(&serving) + .await + .expect("pre-fence side effect remains valid while draining"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; assert!( - !fencing.is_finished(), - "fence must wait for external side effect guard" + store.mark_drained(&token).await.is_err(), + "fenced deletion must wait for a pre-fence external effect lease" ); - drop(guard); - fencing.await.expect("fence task").expect("fence completes"); + assert!(store + .release_serving_write_lease(&serving) + .await + .expect("release")); + store + .mark_drained(&token) + .await + .expect("drain after release"); } #[tokio::test] #[ignore = "requires Postgres"] async fn checkpointed_resume_is_idempotent_and_tombstone_blocks_name_reuse() { let (db, store) = store().await; - let (request, _) = inventoried_request(&db, &store).await; + let (request, inventory) = inventoried_request(&db, &store).await; let host = request.community_host.clone(); store .approve(request.id, "approver", None) @@ -1858,6 +2084,10 @@ mod postgres_tests { fence_generation: Some(generation), ..claim.lease }; + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("freeze destructive storage"); store.mark_drained(&token).await.expect("drain"); store .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) diff --git a/crates/buzz-db/src/error.rs b/crates/buzz-db/src/error.rs index f8b8a2eb56..21e7e40ed6 100644 --- a/crates/buzz-db/src/error.rs +++ b/crates/buzz-db/src/error.rs @@ -45,6 +45,11 @@ pub enum DbError { #[error("invalid data: {0}")] InvalidData(String), + /// A deletion safety invariant is structurally violated and requires + /// operator/code remediation rather than blind retry. + #[error("deletion safety error: {0}")] + DeletionSafety(String), + /// A stored timestamp value could not be interpreted. #[error("invalid timestamp: {0}")] InvalidTimestamp(i64), diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 8022472eac..3e533e26bd 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -1386,12 +1386,19 @@ impl Db { INSERT INTO communities (host) VALUES ($1) ON CONFLICT (lower(host)) DO UPDATE SET host = communities.host + WHERE communities.deletion_state = 'active' + AND communities.deleted_at IS NULL RETURNING id, host, (xmax = 0) AS created "#, ) .bind(normalized_host) - .fetch_one(&self.pool) - .await?; + .fetch_optional(&self.pool) + .await? + .ok_or_else(|| { + DbError::AccessDenied(format!( + "community host {normalized_host:?} is permanently tombstoned" + )) + })?; let id: Uuid = row.try_get("id")?; let host: String = row.try_get("host")?; @@ -1471,6 +1478,8 @@ impl Db { AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' AND c.archived_at IS NULL + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL "#, ) .bind(normalized_host) @@ -1509,6 +1518,8 @@ impl Db { AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' AND lower(c.host) <> lower($3) + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL RETURNING c.id, c.host, c.archived_at"#, ) .bind(normalized_host) @@ -1540,6 +1551,8 @@ impl Db { AND rm.community_id = c.id AND lower(rm.pubkey) = lower($2) AND rm.role = 'owner' + AND c.deletion_state = 'active' + AND c.deleted_at IS NULL RETURNING c.id, c.host"#, ) .bind(normalized_host) diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index b1817c8e15..81fcaf6b5a 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -352,6 +352,7 @@ mod tests { "community_deletion_approvals", "community_deletion_checkpoints", "community_deletion_retention_exceptions", + "community_serving_write_leases", "community_deletion_executor_heartbeats", ] { if normalized[insert_pos..].contains(&format!("'{value}'")) { @@ -1011,6 +1012,7 @@ mod tests { assert!(deletion.contains("CREATE TABLE community_deletion_approvals")); assert!(deletion.contains("CREATE TABLE community_deletion_checkpoints")); assert!(deletion.contains("CREATE TABLE community_deletion_retention_exceptions")); + assert!(deletion.contains("CREATE TABLE community_serving_write_leases")); assert!(deletion.contains("CREATE TABLE community_deletion_executor_heartbeats")); assert!(deletion.contains("CREATE FUNCTION enforce_community_write_fence")); assert!(deletion.contains("CREATE FUNCTION enforce_community_tombstone")); diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml index 6dc812cf4b..0da39834a3 100644 --- a/crates/buzz-deletion/Cargo.toml +++ b/crates/buzz-deletion/Cargo.toml @@ -9,6 +9,7 @@ description = "Durable whole-community deletion engine for Buzz" [dependencies] anyhow = { workspace = true } +thiserror = { workspace = true } axum = { workspace = true } buzz-core = { workspace = true } buzz-db = { workspace = true } @@ -21,3 +22,6 @@ serde_json = { workspace = true } tokio = { workspace = true } tokio-util = { workspace = true } uuid = { workspace = true } + +[dev-dependencies] +sqlx = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 2c1aeb11c7..0def7a7db9 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -2,7 +2,7 @@ #![warn(missing_docs)] //! Shared durable whole-community deletion engine and store adapters. -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -21,12 +21,180 @@ use uuid::Uuid; const DEFAULT_STORAGE_OBJECT_CAP: u64 = 1_000_000; const WORKER_IDLE_POLL: Duration = Duration::from_secs(5); const RETRY_DELAY: Duration = Duration::from_secs(30); +const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10); + +#[cfg(test)] +static TEST_HEARTBEAT_INTERVAL_MS: AtomicU64 = AtomicU64::new(0); + +fn heartbeat_interval() -> Duration { + #[cfg(test)] + { + let milliseconds = TEST_HEARTBEAT_INTERVAL_MS.load(Ordering::Relaxed); + if milliseconds > 0 { + return Duration::from_millis(milliseconds); + } + } + HEARTBEAT_INTERVAL +} + +#[derive(Debug, Clone, thiserror::Error)] +#[error("{message}")] +struct ServingWriteLeaseLost { + message: String, +} /// Return the shared durable deletion store for relay and operator paths. pub fn store(db: &Db) -> DeletionStore { db.deletion_store() } +/// Durable, heartbeated lease for a serving-path external side effect. +pub struct ServingWriteGuard { + store: DeletionStore, + lease: buzz_db::deletion::ServingWriteLease, + cancel: CancellationToken, + lost: CancellationToken, + finished: bool, +} + +impl ServingWriteGuard { + /// Verify this side-effect lease is still current before an irreversible call. + pub async fn verify(&self) -> Result<()> { + if self.lost.is_cancelled() { + return Err(ServingWriteLeaseLost { + message: "serving write lease heartbeat was lost".to_string(), + } + .into()); + } + self.store + .verify_serving_write_lease(&self.lease) + .await + .map_err(|error| ServingWriteLeaseLost { + message: error.to_string(), + })?; + Ok(()) + } + + /// Run an external side effect while observing lease-heartbeat loss. + /// + /// Dropping the operation future on lease loss prevents a stale caller from + /// continuing network I/O after its durable exclusion proof disappears. + pub async fn protect(&self, operation: F) -> Result + where + F: std::future::Future, + { + self.verify().await?; + let output = tokio::select! { + biased; + _ = self.lost.cancelled() => { + return Err(ServingWriteLeaseLost { + message: "serving write lease heartbeat was lost".to_string(), + } + .into()) + } + output = operation => output, + }; + self.verify().await?; + Ok(output) + } + + /// Whether an error represents loss of a durable serving-write lease. + pub fn is_lease_lost(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() + } + + /// Signal fired if the background lease heartbeat fails. + pub fn lost(&self) -> CancellationToken { + self.lost.clone() + } + + /// Begin finalization after the side effect itself has succeeded. + /// + /// This stops lease renewal without releasing the durable row. Call + /// [`Self::finish`] immediately after the result's database finalization so + /// deletion cannot overtake the side effect before its outcome is recorded. + pub fn begin_finalize(&self) { + self.cancel.cancel(); + } + + /// Release the lease after the side effect completes. + pub async fn finish(mut self) -> Result<()> { + self.cancel.cancel(); + let released = self.store.release_serving_write_lease(&self.lease).await?; + self.finished = true; + if !released { + return Err(ServingWriteLeaseLost { + message: "serving write lease was already stale or released".to_string(), + } + .into()); + } + Ok(()) + } +} + +impl Drop for ServingWriteGuard { + fn drop(&mut self) { + self.cancel.cancel(); + if self.finished { + return; + } + let store = self.store.clone(); + let lease = self.lease.clone(); + tokio::spawn(async move { + let _ = store.release_serving_write_lease(&lease).await; + }); + } +} + +/// Acquire a serving-side external-effect lease without holding a pool connection. +pub async fn acquire_serving_write( + db: &Db, + community: buzz_core::CommunityId, + operation: &str, +) -> Result { + let store = store(db); + let owner = default_executor_id(); + let lease = store + .acquire_serving_write_lease(community, operation, &owner, DEFAULT_LEASE_DURATION) + .await?; + let heartbeat_store = store.clone(); + let mut heartbeat_lease = lease.clone(); + let cancel = CancellationToken::new(); + let heartbeat_cancel = cancel.clone(); + let lost = CancellationToken::new(); + let heartbeat_lost = lost.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(heartbeat_interval()); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = heartbeat_cancel.cancelled() => return, + _ = interval.tick() => { + if heartbeat_store + .renew_serving_write_lease( + &mut heartbeat_lease, + DEFAULT_LEASE_DURATION, + ) + .await + .is_err() + { + heartbeat_lost.cancel(); + return; + } + } + } + } + }); + Ok(ServingWriteGuard { + store, + lease, + cancel, + lost, + finished: false, + }) +} + /// CLI-only whole-community deletion commands. #[derive(Subcommand)] pub enum Command { @@ -103,12 +271,79 @@ impl LoopMode { } } +#[derive(Clone)] struct Services { store: DeletionStore, media: Arc, redis: deadpool_redis::Pool, } +#[derive(Debug, thiserror::Error)] +enum EngineError { + #[error("permanent deletion safety failure: {0}")] + Permanent(String), + #[error("transient deletion dependency failure: {0}")] + Transient(String), +} + +#[derive(Debug, thiserror::Error)] +#[error(transparent)] +struct PermanentSource(#[from] anyhow::Error); + +fn permanent(message: impl Into) -> anyhow::Error { + EngineError::Permanent(message.into()).into() +} + +fn permanent_source(error: impl Into) -> anyhow::Error { + PermanentSource(error.into()).into() +} + +fn transient(message: impl Into) -> anyhow::Error { + EngineError::Transient(message.into()).into() +} + +fn is_permanent_error(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause.is::() + || matches!( + cause.downcast_ref::(), + Some(buzz_db::DbError::DeletionSafety(_)) + ) + || cause + .downcast_ref::() + .is_some_and(|error| matches!(error, EngineError::Permanent(_))) + }) +} + +#[derive(Default)] +struct WorkerHealth { + draining: AtomicBool, + dependencies_ready: AtomicBool, + last_heartbeat_epoch: AtomicU64, +} + +impl WorkerHealth { + fn mark_heartbeat(&self) { + self.last_heartbeat_epoch.store( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(), + Ordering::Relaxed, + ); + } + + fn ready(&self) -> bool { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + !self.draining.load(Ordering::Relaxed) + && self.dependencies_ready.load(Ordering::Relaxed) + && now.saturating_sub(self.last_heartbeat_epoch.load(Ordering::Relaxed)) <= 30 + } +} + #[derive(Debug, Serialize)] struct RunOutput { request_id: Uuid, @@ -198,8 +433,7 @@ pub async fn run(command: Command) -> Result { } async fn connect_services() -> Result { - let database_url = std::env::var("DATABASE_URL") - .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); + let database_url = required_env("DATABASE_URL")?; let db = Db::new(&DbConfig { database_url, max_connections: env_parse("BUZZ_DB_POOL_SIZE", 20), @@ -208,16 +442,13 @@ async fn connect_services() -> Result { .await?; let store = store(&db); let media_config = buzz_media::MediaConfig { - s3_endpoint: std::env::var("BUZZ_S3_ENDPOINT") - .unwrap_or_else(|_| "http://localhost:9000".to_string()), - s3_access_key: std::env::var("BUZZ_S3_ACCESS_KEY") - .unwrap_or_else(|_| "buzz_dev".to_string()), - s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY") - .unwrap_or_else(|_| "buzz_dev_secret".to_string()), - s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()), + s3_endpoint: required_env("BUZZ_S3_ENDPOINT")?, + s3_access_key: required_env("BUZZ_S3_ACCESS_KEY")?, + s3_secret_key: required_env("BUZZ_S3_SECRET_KEY")?, + s3_bucket: required_env("BUZZ_S3_BUCKET")?, s3_region: std::env::var("BUZZ_S3_REGION") .or_else(|_| std::env::var("AWS_REGION")) - .unwrap_or_else(|_| "us-east-1".to_string()), + .map_err(|_| anyhow::anyhow!("BUZZ_S3_REGION or AWS_REGION is required"))?, s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE") .unwrap_or_else(|_| "path".to_string()) .parse() @@ -232,8 +463,7 @@ async fn connect_services() -> Result { upload_port_header: None, }; let media = Arc::new(MediaStorage::new(&media_config)?); - let redis_url = - std::env::var("REDIS_URL").unwrap_or_else(|_| "redis://localhost:6379".to_string()); + let redis_url = required_env("REDIS_URL")?; let mut redis_config = deadpool_redis::Config::from_url(&redis_url); redis_config.pool = Some(deadpool_redis::PoolConfig::new(env_parse( "BUZZ_REDIS_POOL_SIZE", @@ -249,6 +479,14 @@ async fn connect_services() -> Result { }) } +fn required_env(name: &str) -> Result { + std::env::var(name) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + .ok_or_else(|| anyhow::anyhow!("{name} is required for community deletion")) +} + fn env_parse(name: &str, default: T) -> T where T: std::str::FromStr, @@ -259,6 +497,12 @@ where .unwrap_or(default) } +fn storage_taxonomy_matches(approved: &StorageManifest, live: &StorageManifest) -> bool { + approved.version == live.version + && approved.unknown_keys == live.unknown_keys + && approved.unsupported_version_keys == live.unsupported_version_keys +} + async fn build_inventory( services: &Services, request: &DeletionRequest, @@ -315,8 +559,13 @@ async fn run_loop( executor_id: String, ) -> Result { let shutdown = shutdown_token(); - let _health = if mode == LoopMode::Worker { - Some(spawn_worker_health(shutdown.clone()).await?) + let health = Arc::new(WorkerHealth::default()); + health.mark_heartbeat(); + health + .dependencies_ready + .store(dependencies_ready(&services).await, Ordering::Relaxed); + let _health_task = if mode == LoopMode::Worker { + Some(spawn_worker_health(shutdown.clone(), Arc::clone(&health)).await?) } else { None }; @@ -349,7 +598,14 @@ async fn run_loop( LoopMode::Worker => { tokio::select! { _ = shutdown.cancelled() => continue, - _ = tokio::time::sleep(WORKER_IDLE_POLL) => continue, + _ = tokio::time::sleep(WORKER_IDLE_POLL) => { + health.dependencies_ready.store( + dependencies_ready(&services).await, + Ordering::Relaxed, + ); + health.mark_heartbeat(); + continue; + }, } } LoopMode::Run if !ran => anyhow::bail!( @@ -359,7 +615,7 @@ async fn run_loop( } }; ran = true; - let output = execute_claim(&services, mode, claim, &shutdown).await?; + let output = execute_claim(&services, mode, claim, &shutdown, Arc::clone(&health)).await?; print_json(&output)?; if mode == LoopMode::Run || shutdown.is_cancelled() { return Ok(i32::from(output.blocked_reason.is_some())); @@ -372,6 +628,7 @@ async fn execute_claim( mode: LoopMode, mut claim: ClaimedDeletion, shutdown: &CancellationToken, + health: Arc, ) -> Result { let token = claim.lease.clone(); loop { @@ -391,12 +648,15 @@ async fn execute_claim( .store .heartbeat(&token, mode.as_str(), DEFAULT_LEASE_DURATION, false) .await?; - let stage_result = execute_stage(services, &claim).await; + health.dependencies_ready.store(true, Ordering::Relaxed); + health.mark_heartbeat(); + let stage_result = + run_stage_with_heartbeat(services, mode, &claim, shutdown, Arc::clone(&health)).await; match stage_result { Ok(()) => {} Err(error) => { let message = format!("{error:#}"); - if is_permanent_failure(&message) { + if is_permanent_error(&error) { services .store .block(&token, claim.request.stage, "stage", &message) @@ -420,59 +680,198 @@ async fn execute_claim( } } +async fn dependencies_ready(services: &Services) -> bool { + if !services.store.ping().await { + return false; + } + let redis_ok = match services.redis.get().await { + Ok(mut connection) => tokio::time::timeout( + Duration::from_secs(5), + redis::cmd("PING").query_async::(&mut *connection), + ) + .await + .is_ok_and(|result| result.is_ok()), + Err(_) => false, + }; + let storage_ok = tokio::time::timeout(Duration::from_secs(5), services.media.ping()) + .await + .is_ok_and(|result| result.is_ok()); + redis_ok && storage_ok +} + +async fn run_stage_with_heartbeat( + services: &Services, + mode: LoopMode, + claim: &ClaimedDeletion, + shutdown: &CancellationToken, + health: Arc, +) -> Result<()> { + let heartbeat_services = services.clone(); + let heartbeat_token = claim.lease.clone(); + let heartbeat_mode = mode.as_str(); + let heartbeat_shutdown = CancellationToken::new(); + let heartbeat_cancel = heartbeat_shutdown.clone(); + let heartbeat_health = Arc::clone(&health); + let heartbeat_error = CancellationToken::new(); + let heartbeat_error_signal = heartbeat_error.clone(); + let heartbeat = tokio::spawn(async move { + let mut interval = tokio::time::interval(heartbeat_interval()); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + interval.tick().await; + loop { + tokio::select! { + _ = heartbeat_cancel.cancelled() => return, + _ = interval.tick() => { + if heartbeat_services + .store + .heartbeat( + &heartbeat_token, + heartbeat_mode, + DEFAULT_LEASE_DURATION, + false, + ) + .await + .is_err() + { + heartbeat_error_signal.cancel(); + return; + } + heartbeat_health.mark_heartbeat(); + } + } + } + }); + + let stage = tokio::select! { + _ = shutdown.cancelled() => Err(anyhow::anyhow!("executor shutdown requested")), + _ = heartbeat_error.cancelled() => Err(anyhow::anyhow!("deletion lease heartbeat failed")), + result = execute_stage(services, claim) => result, + }; + heartbeat_shutdown.cancel(); + match heartbeat.await { + Ok(()) => stage, + Err(error) => Err(anyhow::anyhow!("deletion heartbeat task failed: {error}")), + } +} + +async fn guarded_external_step( + services: &Services, + token: &LeaseToken, + stage: DeletionStage, + operation: F, +) -> Result<()> +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + services.store.verify_execution_token(token, stage).await?; + operation().await?; + services.store.verify_execution_token(token, stage).await?; + Ok(()) +} + async fn execute_stage(services: &Services, claim: &ClaimedDeletion) -> Result<()> { let request = &claim.request; let token = token_with_current_fence(&claim.lease, request); match request.stage { DeletionStage::Approved => { - // Rebuild inventory immediately before the first destructive stage - // and require byte-equivalence with the approved frozen inventory. - let live = build_inventory(services, request).await?; + // Approval binds immutable catalog + key taxonomy. Live row counts + // and tenant binding keys are deliberately not equality-bound until + // the durable fence closes all writers. + let live_schema = services + .store + .inventory_schema(request.community_id) + .await?; let frozen: FrozenInventory = serde_json::from_value( request .inventory_manifest .clone() - .context("approved request has no frozen inventory")?, - )?; - if live != frozen { - anyhow::bail!("approved inventory drifted before fencing"); + .ok_or_else(|| permanent("approved request has no frozen inventory"))?, + ) + .map_err(permanent_source)?; + if live_schema != frozen.schema { + return Err(permanent( + "approved structural catalog drifted before fencing", + )); + } + let live_storage = build_storage_manifest(services, request).await?; + if !storage_taxonomy_matches(&frozen.storage, &live_storage) { + return Err(permanent( + "approved storage taxonomy drifted before fencing", + )); } services.store.fence(&token).await?; } DeletionStage::Fenced => { publish_disconnect_community(&services.redis, request.community_id).await?; - services.store.mark_drained(&token).await?; + let destructive = match request.destructive_storage_manifest.clone() { + Some(value) => serde_json::from_value(value)?, + None => { + let manifest = build_storage_manifest(services, request).await?; + services + .store + .freeze_destructive_storage_manifest(&token, &manifest) + .await?; + manifest + } + }; + buzz_db::deletion::validate_storage_manifest(&destructive)?; + if services + .store + .serving_writes_drained(request.community_id) + .await? + { + services.store.mark_drained(&token).await?; + } else { + return Err(transient("serving writes have not drained")); + } } DeletionStage::Drained => { let storage: StorageManifest = serde_json::from_value( request - .storage_manifest + .destructive_storage_manifest .clone() - .context("request has no frozen storage manifest")?, + .context("request has no post-fence destructive storage manifest")?, )?; buzz_db::deletion::validate_storage_manifest(&storage)?; - // Re-run the exhaustive bucket taxonomy after fencing. Any key that - // appeared after approval is drift and blocks before a delete. let live_storage = build_storage_manifest(services, request).await?; if live_storage != storage { - anyhow::bail!("approved storage inventory drifted before binding removal"); + return Err(permanent( + "post-fence storage inventory drifted before binding removal", + )); } for key in &storage.tenant_keys { match services.media.inspect_current_version(key).await? { CurrentObjectVersion::Present { version_id: None } => {} CurrentObjectVersion::Present { version_id: Some(_), - } => anyhow::bail!("unsupported object version appeared after approval: {key}"), + } => { + return Err(permanent(format!( + "unsupported object version after fence: {key}" + ))) + } CurrentObjectVersion::Missing => { - anyhow::bail!("approved object binding disappeared before deletion: {key}") + return Err(permanent(format!( + "fenced object binding disappeared before deletion: {key}" + ))) } } } - services.media.delete_bindings(&storage.tenant_keys).await?; for key in &storage.tenant_keys { - if services.media.head(key).await? { - anyhow::bail!("object binding still exists after delete: {key}"); - } + guarded_external_step(services, &token, DeletionStage::Drained, || async { + services.media.delete(key).await?; + Ok(()) + }) + .await?; + guarded_external_step(services, &token, DeletionStage::Drained, || async { + if services.media.head(key).await? { + return Err(transient(format!( + "object binding still exists after delete: {key}" + ))); + } + Ok(()) + }) + .await?; } services .store @@ -486,7 +885,15 @@ async fn execute_stage(services: &Services, claim: &ClaimedDeletion) -> Result<( services.store.purge_postgres(&token).await?; } DeletionStage::PostgresPurged => { + services + .store + .verify_execution_token(&token, DeletionStage::PostgresPurged) + .await?; let deleted = purge_redis_namespace(&services.redis, request.community_id).await?; + services + .store + .verify_execution_token(&token, DeletionStage::PostgresPurged) + .await?; services .store .mark_cache_purged(&token, serde_json::json!({"deleted_keys": deleted})) @@ -543,13 +950,15 @@ fn token_with_current_fence(token: &LeaseToken, request: &DeletionRequest) -> Le async fn verify_storage_absence(services: &Services, request: &DeletionRequest) -> Result<()> { let storage: StorageManifest = serde_json::from_value( request - .storage_manifest + .destructive_storage_manifest .clone() - .context("request has no frozen storage manifest")?, + .context("request has no post-fence destructive storage manifest")?, )?; for key in &storage.tenant_keys { if services.media.head(key).await? { - anyhow::bail!("logical verification found object binding: {key}"); + return Err(transient(format!( + "logical verification found object binding: {key}" + ))); } } Ok(()) @@ -601,24 +1010,50 @@ async fn purge_redis_namespace( Ok(deleted) } +fn scan_proves_absence(pages: &[(u64, Vec)]) -> bool { + pages.last().is_some_and(|(cursor, _)| *cursor == 0) + && pages.iter().all(|(_, keys)| keys.is_empty()) +} + +async fn scan_redis_namespace( + connection: &mut deadpool_redis::Connection, + pattern: &str, +) -> Result)>> { + let mut cursor = 0u64; + let mut pages = Vec::new(); + loop { + let page: (u64, Vec) = redis::cmd("SCAN") + .arg(cursor) + .arg("MATCH") + .arg(pattern) + .arg("COUNT") + .arg(1000) + .query_async(&mut **connection) + .await?; + cursor = page.0; + pages.push(page); + if cursor == 0 { + return Ok(pages); + } + } +} + async fn verify_redis_absence( pool: &deadpool_redis::Pool, community: buzz_core::CommunityId, ) -> Result<()> { let mut connection = pool.get().await?; let pattern = format!("buzz:{community}:*"); - let (_cursor, keys): (u64, Vec) = redis::cmd("SCAN") - .arg(0) - .arg("MATCH") - .arg(pattern) - .arg("COUNT") - .arg(1) - .query_async(&mut *connection) - .await?; - if keys.is_empty() { + // SCAN is weakly consistent. Two complete empty passes ensure a cursor + // rollover or concurrent expiry cannot make one sparse pass look absent. + let first = scan_redis_namespace(&mut connection, &pattern).await?; + let second = scan_redis_namespace(&mut connection, &pattern).await?; + if scan_proves_absence(&first) && scan_proves_absence(&second) { Ok(()) } else { - anyhow::bail!("logical verification found Redis key: {}", keys[0]) + Err(transient( + "logical verification found a Redis namespace key", + )) } } @@ -635,32 +1070,33 @@ fn default_executor_id() -> String { format!("{hostname}:{}", std::process::id()) } -async fn spawn_worker_health(shutdown: CancellationToken) -> Result> { +async fn spawn_worker_health( + shutdown: CancellationToken, + health: Arc, +) -> Result> { let address = std::env::var("BUZZ_DELETION_HEALTH_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".to_string()); let listener = tokio::net::TcpListener::bind(&address) .await .with_context(|| format!("bind deletion worker health endpoint {address}"))?; - let draining = Arc::new(AtomicBool::new(false)); - let signal = Arc::clone(&draining); + let signal = Arc::clone(&health); let cancel = shutdown.clone(); tokio::spawn(async move { cancel.cancelled().await; - signal.store(true, Ordering::Relaxed); + signal.draining.store(true, Ordering::Relaxed); }); let router = axum::Router::new() .route("/_liveness", axum::routing::get(|| async { "ok" })) .route( "/_readiness", axum::routing::get({ - let draining = Arc::clone(&draining); move || { - let draining = Arc::clone(&draining); + let health = Arc::clone(&health); async move { - if draining.load(Ordering::Relaxed) { - (axum::http::StatusCode::SERVICE_UNAVAILABLE, "draining") - } else { + if health.ready() { (axum::http::StatusCode::OK, "ready") + } else { + (axum::http::StatusCode::SERVICE_UNAVAILABLE, "not_ready") } } } @@ -701,20 +1137,6 @@ fn shutdown_token() -> CancellationToken { token } -fn is_permanent_failure(message: &str) -> bool { - [ - "drift", - "unknown object-store keys", - "unsupported object", - "catalog", - "write-fence", - "approved inventory", - "tombstone/fence", - ] - .iter() - .any(|needle| message.contains(needle)) -} - fn run_output(request: DeletionRequest) -> RunOutput { RunOutput { request_id: request.id, @@ -733,10 +1155,75 @@ mod tests { use super::*; #[test] - fn permanent_failures_are_narrow_and_fail_closed() { - assert!(is_permanent_failure("community deletion catalog drift")); - assert!(is_permanent_failure("unknown object-store keys")); - assert!(!is_permanent_failure("temporary Redis connection reset")); + fn permanent_failures_are_typed_not_string_classified() { + let permanent_error = permanent("catalog drift"); + let transient_error = transient("temporary catalog service reset"); + let nested = permanent_source(anyhow::anyhow!("schema mismatch")).context("outer"); + let db_permanent = anyhow::Error::from(buzz_db::DbError::DeletionSafety( + "typed catalog drift".to_string(), + )); + let db_transient = anyhow::Error::from(buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut)); + assert!(is_permanent_error(&permanent_error)); + assert!(is_permanent_error(&nested)); + assert!(is_permanent_error(&db_permanent)); + assert!(!is_permanent_error(&transient_error)); + assert!(!is_permanent_error(&db_transient)); + } + + #[test] + fn redis_absence_requires_terminal_cursor_and_all_pages_empty() { + assert!(!scan_proves_absence(&[(9, Vec::new())])); + assert!(!scan_proves_absence(&[ + (9, Vec::new()), + (0, vec!["buzz:tenant:late".to_string()]), + ])); + assert!(scan_proves_absence(&[(9, Vec::new()), (0, Vec::new())])); + } + + #[tokio::test] + async fn serving_guard_cancels_protected_operation_when_heartbeat_is_lost() { + let database_url = match std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + { + Ok(url) => url, + Err(_) => return, + }; + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect serving guard test DB"); + let db = Db::from_pool(pool.clone()); + db.migrate().await.expect("migrate serving guard test DB"); + let community = db + .ensure_configured_community(&format!( + "serving-guard-{}.example", + Uuid::new_v4().simple() + )) + .await + .expect("create test community") + .id; + TEST_HEARTBEAT_INTERVAL_MS.store(10, Ordering::Relaxed); + let guard = acquire_serving_write(&db, community, "test_cancel") + .await + .expect("serving guard"); + sqlx::query("DELETE FROM community_serving_write_leases WHERE community_id = $1") + .bind(community.as_uuid()) + .execute(&pool) + .await + .expect("force heartbeat failure"); + let completed = Arc::new(AtomicBool::new(false)); + let operation_completed = Arc::clone(&completed); + let result = guard + .protect(async move { + tokio::time::sleep(Duration::from_secs(1)).await; + operation_completed.store(true, Ordering::Relaxed); + }) + .await; + TEST_HEARTBEAT_INTERVAL_MS.store(0, Ordering::Relaxed); + assert!(result.is_err(), "lease loss must reject the operation"); + assert!( + !completed.load(Ordering::Relaxed), + "lease loss must cancel the protected operation future" + ); } #[tokio::test] diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index fa5e54e6b1..67c117e8df 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -299,6 +299,11 @@ impl MediaStorage { .map(|m| m.mime_type) } + /// Probe object-store connectivity and bucket access. + pub async fn ping(&self) -> Result<(), MediaError> { + self.list_page(None, 1).await.map(|_| ()) + } + /// One page of a full-bucket listing, for the storage sweep. Wraps /// rust-s3's manual `list_page` (NOT the auto-paginating `list`, which /// has no cap) and converts the result into the storage-agnostic diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index 6f2eb81da5..48ea42a371 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -1826,9 +1826,12 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { // An already-running receive-pack may cross the durable fence after // request admission. Revalidate immediately before object-store CAS; DB // trigger fencing alone cannot roll back an S3 pointer mutation. - let _serving_write = match buzz_deletion::store(&state.db) - .begin_serving_write(ctx.tenant.community()) - .await + let serving_write = match buzz_deletion::acquire_serving_write( + &state.db, + ctx.tenant.community(), + "git_publish", + ) + .await { Ok(guard) => guard, Err(error) => { @@ -1841,10 +1844,20 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { } }; + if let Err(error) = serving_write.verify().await { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push lost community serving lease"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost", + ) + .into_response(); + } + // Step 7 (CAS). The PushContext binds `parent_state` (observed at // hydrate) to the CAS predicate here — no re-reading of the pointer - // between hydrate and CAS. - let success = match cas_publish( + // between hydrate and CAS. Observe serving-lease loss throughout the + // potentially long upload/CAS operation, not only at its boundaries. + let publish = cas_publish( &state.git_store, &ctx.tenant, ctx.repo_handle.path(), @@ -1856,72 +1869,85 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { max_pack_bytes: state.config.git_max_pack_bytes, max_repo_bytes: state.config.git_max_repo_bytes, }, - ) - .await - { - Ok(s) => s, - Err(CasError::Conflict { - winner_manifest_key, - .. - }) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - winner = %winner_manifest_key, - "push lost CAS race; tempdir dropped, returning 409" - ); - return ( - StatusCode::CONFLICT, - "push superseded by a concurrent writer; pull and retry", - ) - .into_response(); - } - Err(CasError::ManifestInvalid(e)) => { - // 4xx-class: the workspace produced refs/HEAD/oids the - // manifest validator rejects (unsafe refname, malformed oid, - // empty head, malformed parent). Pre-CAS — no pointer was - // written. - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push rejected: manifest validation failed" - ); - return ( - StatusCode::BAD_REQUEST, - "push produced invalid manifest state", - ) - .into_response(); - } - Err(CasError::ResourceLimit(e)) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push rejected: repo exceeds relay resource limits" - ); + ); + let success = match serving_write.protect(publish).await { + Ok(result) => match result { + Ok(s) => s, + Err(CasError::Conflict { + winner_manifest_key, + .. + }) => { + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + winner = %winner_manifest_key, + "push lost CAS race; tempdir dropped, returning 409" + ); + return ( + StatusCode::CONFLICT, + "push superseded by a concurrent writer; pull and retry", + ) + .into_response(); + } + Err(CasError::ManifestInvalid(e)) => { + // 4xx-class: the workspace produced refs/HEAD/oids the + // manifest validator rejects (unsafe refname, malformed oid, + // empty head, malformed parent). Pre-CAS — no pointer was + // written. + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push rejected: manifest validation failed" + ); + return ( + StatusCode::BAD_REQUEST, + "push produced invalid manifest state", + ) + .into_response(); + } + Err(CasError::ResourceLimit(e)) => { + warn!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push rejected: repo exceeds relay resource limits" + ); + return ( + StatusCode::PAYLOAD_TOO_LARGE, + "repository exceeds relay resource limits", + ) + .into_response(); + } + Err(e) => { + // 5xx-class: ManifestReadFailed (parent corruption), + // Backend, PackCapture. The tempdir drops on scope exit; no + // pointer was written (or, on rare ManifestReadFailed during + // winner-fetch, the winner is already installed and the + // loser's data is unrelated). + error!( + owner = %ctx.owner, + repo = %ctx.repo, + error = %e, + "push failed pre-response" + ); + return (StatusCode::INTERNAL_SERVER_ERROR, "git backend error").into_response(); + } + }, + Err(error) => { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "push lost community serving lease during CAS publish"); return ( - StatusCode::PAYLOAD_TOO_LARGE, - "repository exceeds relay resource limits", + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost", ) .into_response(); } - Err(e) => { - // 5xx-class: ManifestReadFailed (parent corruption), - // Backend, PackCapture. The tempdir drops on scope exit; no - // pointer was written (or, on rare ManifestReadFailed during - // winner-fetch, the winner is already installed and the - // loser's data is unrelated). - error!( - owner = %ctx.owner, - repo = %ctx.repo, - error = %e, - "push failed pre-response" - ); - return (StatusCode::INTERNAL_SERVER_ERROR, "git backend error").into_response(); - } }; + if let Err(error) = serving_write.finish().await { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "failed to release community serving lease after push"); + } + // Derived after CAS: kind:30618 ref-state event over the *committed* // manifest's refs/head. Spec §Implementation Correspondence: // "kind:30618 is derived after CAS, never the commit." We emit only diff --git a/crates/buzz-relay/src/api/invites.rs b/crates/buzz-relay/src/api/invites.rs index 6104171cca..60b7caf2af 100644 --- a/crates/buzz-relay/src/api/invites.rs +++ b/crates/buzz-relay/src/api/invites.rs @@ -305,7 +305,9 @@ pub async fn mint_invite( .mint_relay_invite(tenant.community(), &sender_hex, ttl, max_uses) .await .map_err(|error| match error { - buzz_db::DbError::InvalidData(message) => api_error(StatusCode::BAD_REQUEST, &message), + buzz_db::DbError::InvalidData(message) | buzz_db::DbError::DeletionSafety(message) => { + api_error(StatusCode::BAD_REQUEST, &message) + } error => internal_error(&format!("invite mint: {error}")), })?; diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 41c10e8e63..3f84405827 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -310,10 +310,10 @@ pub async fn upload_blob( ) -> Result, MediaError> { let attribution = upload_attribution(&state, &auth, &headers).await; - let _serving_write = buzz_deletion::store(&state.db) - .begin_serving_write(auth.tenant.community()) - .await - .map_err(|_| MediaError::RelayMembershipRequired)?; + let serving_write = + buzz_deletion::acquire_serving_write(&state.db, auth.tenant.community(), "media_upload") + .await + .map_err(|_| MediaError::RelayMembershipRequired)?; if auth.route_mode == UploadRouteMode::LegacyMedia { metrics::counter!("buzz_media_legacy_upload_route_total").increment(1); @@ -340,69 +340,89 @@ pub async fn upload_blob( } let replay = futures_util::stream::iter(replay_chunks.into_iter().map(Ok)).chain(source); - let mut descriptor = if should_stream_as_video(&sniff) { - // Video path: stream body directly to disk — never fully buffered in RAM. - let content_length = headers - .get("content-length") - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()); - buzz_media::process_video_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - replay, - content_length, - attribution, - ) - .await? - } else { - // Non-video path: buffer the body (bounded by the larger of the image - // and generic-file caps), then decide image-vs-generic by sniffed MIME. - // Images go through the thumbnailing pipeline; non-media attachments - // (docs, archives, text, data) take the generic file path and are - // served as downloads. Recognized audio/video cannot fall through it. - let max = state - .config - .media - .max_image_bytes - .max(state.config.media.max_file_bytes); - let bytes = axum::body::to_bytes(axum::body::Body::from_stream(replay), max as usize) - .await - .map_err(|_| MediaError::FileTooLarge { size: 0, max })?; - - let is_image = matches!( - infer::get(&bytes).map(|t| t.mime_type()), - Some("image/jpeg" | "image/png" | "image/gif" | "image/webp") - ); + serving_write + .verify() + .await + .map_err(|_| MediaError::RelayMembershipRequired)?; - if is_image { - buzz_media::process_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - bytes, - attribution, - ) - .await? - } else if auth.route_mode == UploadRouteMode::LegacyMedia { - let mime = infer::get(&bytes) - .map(|kind| kind.mime_type().to_string()) - .unwrap_or_else(|| "application/octet-stream".to_string()); - return Err(MediaError::DisallowedContentType(mime)); - } else { - buzz_media::process_file_upload( - &state.media_storage, - &state.config.media, - &auth.tenant, - &auth.auth_event, - bytes, - attribution, - ) - .await? - } - }; + let mut descriptor = serving_write + .protect(async { + Ok(if should_stream_as_video(&sniff) { + // Video path: stream body directly to disk — never fully buffered in RAM. + let content_length = headers + .get("content-length") + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()); + buzz_media::process_video_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + replay, + content_length, + attribution, + ) + .await? + } else { + // Non-video path: buffer the body (bounded by the larger of the image + // and generic-file caps), then decide image-vs-generic by sniffed MIME. + // Images go through the thumbnailing pipeline; non-media attachments + // (docs, archives, text, data) take the generic file path and are + // served as downloads. Recognized audio/video cannot fall through it. + let max = state + .config + .media + .max_image_bytes + .max(state.config.media.max_file_bytes); + let bytes = + axum::body::to_bytes(axum::body::Body::from_stream(replay), max as usize) + .await + .map_err(|_| MediaError::FileTooLarge { size: 0, max })?; + + let is_image = matches!( + infer::get(&bytes).map(|t| t.mime_type()), + Some("image/jpeg" | "image/png" | "image/gif" | "image/webp") + ); + + if is_image { + buzz_media::process_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + bytes, + attribution, + ) + .await? + } else if auth.route_mode == UploadRouteMode::LegacyMedia { + let mime = infer::get(&bytes) + .map(|kind| kind.mime_type().to_string()) + .unwrap_or_else(|| "application/octet-stream".to_string()); + return Err(MediaError::DisallowedContentType(mime)); + } else { + buzz_media::process_file_upload( + &state.media_storage, + &state.config.media, + &auth.tenant, + &auth.auth_event, + bytes, + attribution, + ) + .await? + } + }) + }) + .await + .map_err(|error| { + if buzz_deletion::ServingWriteGuard::is_lease_lost(&error) { + MediaError::RelayMembershipRequired + } else { + match error.downcast::() { + Ok(error) => error, + Err(_) => MediaError::Internal, + } + } + })??; rewrite_descriptor_urls_for_tenant( &mut descriptor, @@ -446,6 +466,10 @@ pub async fn upload_blob( } } + serving_write + .finish() + .await + .map_err(|_| MediaError::RelayMembershipRequired)?; Ok(Json(descriptor)) } diff --git a/crates/buzz-relay/src/api/mesh_demo.rs b/crates/buzz-relay/src/api/mesh_demo.rs index 8649b97671..902366f0d1 100644 --- a/crates/buzz-relay/src/api/mesh_demo.rs +++ b/crates/buzz-relay/src/api/mesh_demo.rs @@ -310,13 +310,28 @@ mod tests { owner_runtime, ); let from = hello.sender; - let inbound = router.accept_inbound(from, hello, stream).await.unwrap(); - crate::mesh_boot::run_demo_echo( - echo_directory, - inbound, - std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), - ) - .await; + let mut inbound = router.accept_inbound(from, hello, stream).await.unwrap(); + let frame = inbound + .stream + .recv_validated(&echo_directory) + .await + .unwrap() + .unwrap(); + let ReliableFrame::Data(payload) = frame else { + panic!("expected data"); + }; + let community = inbound.stream.community_id().unwrap(); + inbound + .stream + .send_bytes(community, &payload) + .await + .unwrap(); + // Keep both endpoints and the owner stream alive until the caller has + // consumed the echo. Dropping the last endpoint clone closes the QUIC + // connection and used to make this production-path regression test + // deterministically time out before the return frame was read. + let _owner_endpoint_guard = accept_endpoint; + tokio::time::sleep(ECHO_TIMEOUT).await; }); // Forwarding side: join the same session through the demo core. @@ -341,5 +356,7 @@ mod tests { assert_eq!(body["outcome"], "forwarded"); assert_eq!(body["echoed_payload"], "mesh echo evidence"); owner_task.abort(); + drop(local_endpoint); + drop(owner_endpoint); } } diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index c6d4655365..3a63727986 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -280,18 +280,6 @@ pub(crate) async fn fan_out_event_to_local_subscribers( /// Fan out one event received from Redis pub/sub to this relay's local subscribers. #[tracing::instrument(skip_all)] pub async fn fan_out_pubsub_event(state: &Arc, channel_event: buzz_pubsub::ChannelEvent) { - // Redis can carry an event published just before the deletion fence. Do not - // deliver stale post-fence fan-out to a community that is no longer active. - if !matches!( - state - .db - .is_community_active(channel_event.community_id) - .await, - Ok(true) - ) { - return; - } - // The Redis topic carries the tenant-local routing scope explicitly: // `Channel(id)` for a per-channel event, `Global` for a channel-less one. // Convert back to the `Option` channel id `fan_out()` indexes on — @@ -679,22 +667,6 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc guard, - Err(error) => { - reject("restricted"); - conn.send(RelayMessage::ok( - &event_id_hex, - false, - &format!("restricted: community writes are fenced: {error}"), - )); - return; - } - }; - if kind_u32 == buzz_core::kind::KIND_AUTH { reject("invalid"); conn.send(RelayMessage::ok( @@ -733,16 +705,47 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc guard, + Err(error) => { + reject("restricted"); + conn.send(RelayMessage::ok( + &event_id_hex, + false, + &format!("restricted: community writes are fenced: {error}"), + )); + return; + } + }; + let handled = serving_write + .protect(handle_ephemeral_event( + event, + conn_id, + &event_id_hex, + pubkey_bytes, + auth_pubkey, + Arc::clone(&conn), + state, + )) + .await; + if let Err(error) = handled { + reject("restricted"); + conn.send(RelayMessage::ok( + &event_id_hex, + false, + &format!("restricted: community write lease lost: {error}"), + )); + return; + } + if let Err(error) = serving_write.finish().await { + tracing::warn!(%error, event_id = %event_id_hex, "failed to release ephemeral-event serving lease"); + } return; } diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index de39169d68..cd9f20b5f4 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -1863,13 +1863,6 @@ async fn ingest_event_inner( let kind_u32 = event_kind_u32(&event); debug!(event_id = %event_id_hex, kind = kind_u32, "ingest_event"); - buzz_deletion::store(&state.db) - .assert_serving_write_allowed(tenant.community()) - .await - .map_err(|error| { - IngestError::Rejected(format!("restricted: community writes are fenced: {error}")) - })?; - if kind_u32 == KIND_AUTH { return Err(IngestError::Rejected( "invalid: AUTH events cannot be submitted".into(), diff --git a/crates/buzz-relay/src/mesh_boot.rs b/crates/buzz-relay/src/mesh_boot.rs index 4f452fed89..cd7c427c72 100644 --- a/crates/buzz-relay/src/mesh_boot.rs +++ b/crates/buzz-relay/src/mesh_boot.rs @@ -536,18 +536,15 @@ mod tests { .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .unwrap(); let keys = nostr::Keys::generate(); - let db_pool = sqlx::postgres::PgPoolOptions::new() - .connect_lazy("postgres://buzz:buzz_dev@localhost:5432/buzz") - .unwrap(); - let handle = boot_mesh( - &config, - pool, - buzz_db::Db::from_pool(db_pool), - &keys, - Arc::new(AtomicBool::new(false)), - ) - .await - .expect("off path is never an error"); + let db = buzz_db::Db::from_pool( + sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_lazy("postgres://unused:unused@127.0.0.1:1/unused") + .expect("lazy database pool"), + ); + let handle = boot_mesh(&config, pool, db, &keys, Arc::new(AtomicBool::new(false))) + .await + .expect("off path is never an error"); assert!(handle.is_none()); } diff --git a/crates/buzz-relay/src/push_runtime.rs b/crates/buzz-relay/src/push_runtime.rs index 5a261ac912..d6b7f04b91 100644 --- a/crates/buzz-relay/src/push_runtime.rs +++ b/crates/buzz-relay/src/push_runtime.rs @@ -418,9 +418,12 @@ async fn deliver_one( return; } }; - let _serving_write = match buzz_deletion::store(&state.db) - .begin_serving_write(outcome.community) - .await + let serving_write = match buzz_deletion::acquire_serving_write( + &state.db, + outcome.community, + "push_delivery", + ) + .await { Ok(guard) => guard, Err(error) => { @@ -443,7 +446,21 @@ async fn deliver_one( return; } }; - let response = send_gateway_request(http, url, body, auth).await; + if let Err(error) = serving_write.verify().await { + warn!(wake=%outcome.id, %error, "push serving lease lost before delivery"); + return; + } + let response = match serving_write + .protect(send_gateway_request(http, url, body, auth)) + .await + { + Ok(response) => response, + Err(error) => { + warn!(wake=%outcome.id, %error, "push serving lease lost during delivery"); + return; + } + }; + serving_write.begin_finalize(); match response { Ok(r) if r.status().is_success() => match r.json::().await { Ok(DeliveryResponse::Accepted) => { @@ -516,6 +533,9 @@ async fn deliver_one( .await; } } + if let Err(error) = serving_write.finish().await { + warn!(wake=%outcome.id, %error, "failed to release community serving lease after push delivery"); + } } fn delivery_body(endpoint_grant: &str, request_id: uuid::Uuid, expires_at: i64) -> Vec { diff --git a/crates/buzz-relay/src/tunnel/directory.rs b/crates/buzz-relay/src/tunnel/directory.rs index 82542f1c87..42b121e431 100644 --- a/crates/buzz-relay/src/tunnel/directory.rs +++ b/crates/buzz-relay/src/tunnel/directory.rs @@ -180,7 +180,7 @@ pub enum DirectoryError { /// Lease TTL cannot be represented in Redis milliseconds. #[error("lease ttl must be at least 1ms and fit in i64 milliseconds")] InvalidLeaseTtl, - /// Durable community deletion fence rejected a serving write. + /// Durable community deletion fence rejected a Redis mutation. #[error("community write fenced: {0}")] CommunityWriteFenced(String), } @@ -191,8 +191,8 @@ impl SessionDirectory { Self::with_lease_ttl(pool, DEFAULT_LEASE_TTL) } - /// Create a serving directory whose Redis mutations consult the durable - /// community write fence. + /// Create a serving directory whose Redis mutations use durable, + /// heartbeat-backed community write leases. pub fn with_db(pool: deadpool_redis::Pool, db: buzz_db::Db) -> Self { Self { pool, @@ -213,10 +213,9 @@ impl SessionDirectory { async fn begin_serving_write( &self, community_id: CommunityId, - ) -> Result, DirectoryError> { + ) -> Result, DirectoryError> { match &self.db { - Some(db) => buzz_deletion::store(db) - .begin_serving_write(community_id) + Some(db) => buzz_deletion::acquire_serving_write(db, community_id, "session_directory") .await .map(Some) .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string())), @@ -236,11 +235,11 @@ impl SessionDirectory { owner_runtime_id: RuntimeId, profile: Profile, ) -> Result { - let _serving_write = self.begin_serving_write(community_id).await?; + let serving_write = self.begin_serving_write(community_id).await?; let keys = SessionKeys::new(community_id, session_id); let ttl_ms = ttl_ms(self.lease_ttl)?; let mut conn = self.pool.get().await?; - let (status, value, _known_generation): (String, String, String) = + let mutation = async { Script::new(ACQUIRE_SCRIPT) .key(&keys.lease) .key(&keys.generation) @@ -248,8 +247,22 @@ impl SessionDirectory { .arg(profile.as_wire_str()) .arg(ttl_ms) .invoke_async(&mut *conn) - .await?; + .await + }; + let (status, value, _known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let lease = parse_lease(community_id, session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "acquired" => Ok(AcquireResult::Acquired(lease)), "exists" => Ok(AcquireResult::Exists(lease)), @@ -277,19 +290,34 @@ impl SessionDirectory { /// Renew a lease only if the current Redis value exactly matches the /// caller's owner runtime and generation. pub async fn renew(&self, lease: &SessionLease) -> Result { - let _serving_write = self.begin_serving_write(lease.community_id).await?; + let serving_write = self.begin_serving_write(lease.community_id).await?; let keys = SessionKeys::new(lease.community_id, lease.session_id); let ttl_ms = ttl_ms(self.lease_ttl)?; let mut conn = self.pool.get().await?; - let (status, value, known_generation): (String, String, String) = Script::new(RENEW_SCRIPT) - .key(&keys.lease) - .key(&keys.generation) - .arg(lease.owner_runtime_id.to_hex()) - .arg(lease.generation) - .arg(ttl_ms) - .invoke_async(&mut *conn) - .await?; + let mutation = async { + Script::new(RENEW_SCRIPT) + .key(&keys.lease) + .key(&keys.generation) + .arg(lease.owner_runtime_id.to_hex()) + .arg(lease.generation) + .arg(ttl_ms) + .invoke_async(&mut *conn) + .await + }; + let (status, value, known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let current = parse_optional_lease(lease.community_id, lease.session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "renewed" => Ok(RenewResult::Renewed( current.expect("renewed returns lease"), @@ -309,17 +337,32 @@ impl SessionDirectory { /// Release a lease only if the current Redis value exactly matches the /// caller's owner runtime and generation. pub async fn release(&self, lease: &SessionLease) -> Result { + let serving_write = self.begin_serving_write(lease.community_id).await?; let keys = SessionKeys::new(lease.community_id, lease.session_id); let mut conn = self.pool.get().await?; - let (status, value, known_generation): (String, String, String) = + let mutation = async { Script::new(RELEASE_SCRIPT) .key(&keys.lease) .key(&keys.generation) .arg(lease.owner_runtime_id.to_hex()) .arg(lease.generation) .invoke_async(&mut *conn) - .await?; + .await + }; + let (status, value, known_generation): (String, String, String) = match &serving_write { + Some(guard) => guard + .protect(mutation) + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))??, + None => mutation.await?, + }; let current = parse_optional_lease(lease.community_id, lease.session_id, &value)?; + if let Some(guard) = serving_write { + guard + .finish() + .await + .map_err(|error| DirectoryError::CommunityWriteFenced(error.to_string()))?; + } match status.as_str() { "released" => Ok(ReleaseResult::Released( current.expect("released returns lease"), diff --git a/crates/buzz-relay/src/workflow_sink.rs b/crates/buzz-relay/src/workflow_sink.rs index ad474fafa7..97c31c2561 100644 --- a/crates/buzz-relay/src/workflow_sink.rs +++ b/crates/buzz-relay/src/workflow_sink.rs @@ -334,11 +334,6 @@ impl ActionSink for RelayActionSink { broadcast: false, }); - buzz_deletion::store(&state.db) - .assert_serving_write_allowed(tenant.community()) - .await - .map_err(|error| ActionSinkError::Database(error.to_string()))?; - let (stored_event, was_inserted) = state .db .insert_event_with_thread_metadata( diff --git a/crates/buzz-workflow/src/executor.rs b/crates/buzz-workflow/src/executor.rs index 8a52ae67ae..dffa492716 100644 --- a/crates/buzz-workflow/src/executor.rs +++ b/crates/buzz-workflow/src/executor.rs @@ -530,174 +530,198 @@ pub async fn dispatch_action( // Revalidate the durable community fence immediately before every external // side effect (message publish, webhook, delay/resume). A storage failure is // a denial, never permission to continue. - let _serving_write = buzz_deletion::store(&engine.db) - .begin_serving_write(community_id) - .await - .map_err(|error| { - WorkflowError::WebhookError(format!( - "community write fence rejected workflow side effect: {error}" - )) - })?; - - match action { - SendMessage { text, channel } => { - // Look up workflow metadata for destination validation and - // attribution, scoped to the run's community — the same run/workflow - // UUID may exist in another community, so a bare-id lookup could - // load the wrong row and drive a side effect under it. - let wf_run = engine - .db - .get_workflow_run(community_id, run_id) - .await - .map_err(|e| { - WorkflowError::WebhookError(format!( - "SendMessage: failed to load workflow run {run_id}: {e}" - )) - })?; - let workflow = engine - .db - .get_workflow(community_id, wf_run.workflow_id) - .await - .map_err(|e| { - WorkflowError::WebhookError(format!( - "SendMessage: failed to load workflow {}: {e}", - wf_run.workflow_id - )) - })?; - let channel_id = resolve_send_message_channel( - channel.as_deref(), - &trigger_ctx.channel_id, - workflow.channel_id, - )?; - let owner_pubkey_hex = hex::encode(&workflow.owner_pubkey); - - info!( - run_id = %run_id, - step = step_id, - channel = %channel_id, - "SendMessage → {channel_id}: {text}" - ); - - let event_id = engine - .action_sink()? - .send_message(community_id, &channel_id, text, &owner_pubkey_hex) - .await - .map_err(WorkflowError::from)?; - - Ok(StepResult::Completed(serde_json::json!({ - "sent": true, - "event_id": event_id, - }))) - } - - SendDm { to, text: _ } => { - warn!(run_id = %run_id, step = step_id, "SendDm not yet implemented (to={to})"); - // TODO (WF-07): emit DM event. - Err(WorkflowError::NotImplemented("SendDm".into())) - } - - SetChannelTopic { topic: _ } => { - warn!(run_id = %run_id, step = step_id, "SetChannelTopic not yet implemented"); - // TODO (WF-07): update channel topic via DB. - Err(WorkflowError::NotImplemented("SetChannelTopic".into())) - } - - AddReaction { emoji } => { - info!(run_id = %run_id, step = step_id, "AddReaction → :{emoji}:"); - if trigger_ctx.message_id.is_empty() { - return Err(WorkflowError::InvalidDefinition( - "AddReaction: no trigger.message_id available".into(), - )); - } - - #[cfg(feature = "reqwest")] - { - let result = add_reaction_impl(&trigger_ctx.message_id, emoji).await?; - Ok(StepResult::Completed(result)) - } - - #[cfg(not(feature = "reqwest"))] - { - warn!( - run_id = %run_id, - step = step_id, - "AddReaction: reqwest feature not enabled, skipping HTTP call" - ); - Ok(StepResult::Completed( - serde_json::json!({ "added": false, "skipped": true }), + let serving_write = + buzz_deletion::acquire_serving_write(&engine.db, community_id, "workflow_action") + .await + .map_err(|error| { + WorkflowError::WebhookError(format!( + "community write fence rejected workflow side effect: {error}" )) - } - } + })?; - CallWebhook { - url, - method, - headers, - body, - } => { - let method_str = method.as_deref().unwrap_or("POST"); - info!(run_id = %run_id, step = step_id, "CallWebhook → {method_str} {url}"); + serving_write.verify().await.map_err(|error| { + WorkflowError::WebhookError(format!("community write lease lost: {error}")) + })?; - #[cfg(feature = "reqwest")] - { - let result = call_webhook_impl(url, method_str, headers, body).await?; - Ok(StepResult::Completed(result)) - } + let result = serving_write + .protect(async { + match action { + SendMessage { text, channel } => { + // Look up workflow metadata for destination validation and + // attribution, scoped to the run's community — the same run/workflow + // UUID may exist in another community, so a bare-id lookup could + // load the wrong row and drive a side effect under it. + let wf_run = engine + .db + .get_workflow_run(community_id, run_id) + .await + .map_err(|e| { + WorkflowError::WebhookError(format!( + "SendMessage: failed to load workflow run {run_id}: {e}" + )) + })?; + let workflow = engine + .db + .get_workflow(community_id, wf_run.workflow_id) + .await + .map_err(|e| { + WorkflowError::WebhookError(format!( + "SendMessage: failed to load workflow {}: {e}", + wf_run.workflow_id + )) + })?; + let channel_id = resolve_send_message_channel( + channel.as_deref(), + &trigger_ctx.channel_id, + workflow.channel_id, + )?; + let owner_pubkey_hex = hex::encode(&workflow.owner_pubkey); + + info!( + run_id = %run_id, + step = step_id, + channel = %channel_id, + "SendMessage → {channel_id}: {text}" + ); + + let event_id = engine + .action_sink()? + .send_message(community_id, &channel_id, text, &owner_pubkey_hex) + .await + .map_err(WorkflowError::from)?; + + Ok(StepResult::Completed(serde_json::json!({ + "sent": true, + "event_id": event_id, + }))) + } - #[cfg(not(feature = "reqwest"))] - { - // reqwest not enabled — log and return placeholder. - warn!( - run_id = %run_id, step = step_id, - "CallWebhook: reqwest feature not enabled, skipping HTTP call" - ); - let _ = (headers, body); // suppress unused warnings - Ok(StepResult::Completed(serde_json::json!({ - "status": 0, - "body": null, - "skipped": true - }))) - } - } + SendDm { to, text: _ } => { + warn!(run_id = %run_id, step = step_id, "SendDm not yet implemented (to={to})"); + // TODO (WF-07): emit DM event. + Err(WorkflowError::NotImplemented("SendDm".into())) + } - RequestApproval { - from, - message, - timeout, - } => { - let timeout_str = timeout.as_deref().unwrap_or("24h"); - info!( - run_id = %run_id, step = step_id, - "RequestApproval from={from} timeout={timeout_str}: {message}" - ); + SetChannelTopic { topic: _ } => { + warn!(run_id = %run_id, step = step_id, "SetChannelTopic not yet implemented"); + // TODO (WF-07): update channel topic via DB. + Err(WorkflowError::NotImplemented("SetChannelTopic".into())) + } - let token = generate_approval_token(run_id, step_id); + AddReaction { emoji } => { + info!(run_id = %run_id, step = step_id, "AddReaction → :{emoji}:"); + if trigger_ctx.message_id.is_empty() { + Err(WorkflowError::InvalidDefinition( + "AddReaction: no trigger.message_id available".into(), + )) + } else { + #[cfg(feature = "reqwest")] + { + let result = add_reaction_impl(&trigger_ctx.message_id, emoji).await?; + Ok(StepResult::Completed(result)) + } + + #[cfg(not(feature = "reqwest"))] + { + warn!( + run_id = %run_id, + step = step_id, + "AddReaction: reqwest feature not enabled, skipping HTTP call" + ); + Ok(StepResult::Completed( + serde_json::json!({ "added": false, "skipped": true }), + )) + } + } + } - // TODO (WF-08): create approval record in DB, emit kind:46010. - // For now, return Suspended with the token so the caller can persist state. + CallWebhook { + url, + method, + headers, + body, + } => { + let method_str = method.as_deref().unwrap_or("POST"); + info!(run_id = %run_id, step = step_id, "CallWebhook → {method_str} {url}"); + + #[cfg(feature = "reqwest")] + { + let result = call_webhook_impl(url, method_str, headers, body).await?; + Ok(StepResult::Completed(result)) + } - Ok(StepResult::Suspended { - approval_token: token, - }) - } + #[cfg(not(feature = "reqwest"))] + { + // reqwest not enabled — log and return placeholder. + warn!( + run_id = %run_id, step = step_id, + "CallWebhook: reqwest feature not enabled, skipping HTTP call" + ); + let _ = (headers, body); // suppress unused warnings + Ok(StepResult::Completed(serde_json::json!({ + "status": 0, + "body": null, + "skipped": true + }))) + } + } - Delay { duration } => { - let secs = parse_duration_secs(duration)?; - // Cap delay at 270 seconds (4.5 minutes) — must be less than default_timeout_secs (300s) - // to avoid non-deterministic StepTimeout. Long delays (hours/days) - // should use the scheduled resume pattern (future work: WF-09). - const MAX_DELAY_SECS: u64 = 270; - if secs > MAX_DELAY_SECS { - return Err(WorkflowError::InvalidDefinition(format!( - "delay exceeds maximum of {MAX_DELAY_SECS} seconds (got {secs}s); \ + RequestApproval { + from, + message, + timeout, + } => { + let timeout_str = timeout.as_deref().unwrap_or("24h"); + info!( + run_id = %run_id, step = step_id, + "RequestApproval from={from} timeout={timeout_str}: {message}" + ); + + let token = generate_approval_token(run_id, step_id); + + // TODO (WF-08): create approval record in DB, emit kind:46010. + // For now, return Suspended with the token so the caller can persist state. + + Ok(StepResult::Suspended { + approval_token: token, + }) + } + + Delay { duration } => { + let secs = parse_duration_secs(duration)?; + // Cap delay at 270 seconds (4.5 minutes) — must be less than default_timeout_secs (300s) + // to avoid non-deterministic StepTimeout. Long delays (hours/days) + // should use the scheduled resume pattern (future work: WF-09). + const MAX_DELAY_SECS: u64 = 270; + if secs > MAX_DELAY_SECS { + return Err(WorkflowError::InvalidDefinition(format!( + "delay exceeds maximum of {MAX_DELAY_SECS} seconds (got {secs}s); \ use the scheduled resume pattern for long delays" - ))); + ))); + } + info!(run_id = %run_id, step = step_id, "Delay {duration} ({secs}s)"); + tokio::time::sleep(std::time::Duration::from_secs(secs)).await; + Ok(StepResult::Completed( + serde_json::json!({ "slept_secs": secs }), + )) + } } - info!(run_id = %run_id, step = step_id, "Delay {duration} ({secs}s)"); - tokio::time::sleep(std::time::Duration::from_secs(secs)).await; - Ok(StepResult::Completed( - serde_json::json!({ "slept_secs": secs }), - )) + }) + .await + .map_err(|error| { + WorkflowError::WebhookError(format!("community write lease lost: {error}")) + })?; + let release = serving_write.finish().await.map_err(|error| { + WorkflowError::WebhookError(format!("community write lease release failed: {error}")) + }); + match result { + Ok(value) => { + release?; + Ok(value) + } + Err(error) => { + let _ = release; + Err(error) } } } diff --git a/deploy/charts/buzz/templates/deletion-worker-deployment.yaml b/deploy/charts/buzz/templates/deletion-worker-deployment.yaml index 2e32ac8ec1..42f02fb6d9 100644 --- a/deploy/charts/buzz/templates/deletion-worker-deployment.yaml +++ b/deploy/charts/buzz/templates/deletion-worker-deployment.yaml @@ -72,13 +72,11 @@ spec: secretKeyRef: name: {{ .Values.deletionWorker.existingSecret }} key: BUZZ_S3_ACCESS_KEY - optional: true - name: BUZZ_S3_SECRET_KEY valueFrom: secretKeyRef: name: {{ .Values.deletionWorker.existingSecret }} key: BUZZ_S3_SECRET_KEY - optional: true {{- with .Values.deletionWorker.extraEnv }} {{- toYaml . | nindent 12 }} {{- end }} diff --git a/deploy/charts/buzz/tests/deletion_worker_test.yaml b/deploy/charts/buzz/tests/deletion_worker_test.yaml index e76604954c..53730b0a94 100644 --- a/deploy/charts/buzz/tests/deletion_worker_test.yaml +++ b/deploy/charts/buzz/tests/deletion_worker_test.yaml @@ -43,3 +43,7 @@ tests: - equal: path: spec.template.spec.containers[0].readinessProbe.httpGet.path value: /_readiness + - notExists: + path: spec.template.spec.containers[0].env[8].valueFrom.secretKeyRef.optional + - notExists: + path: spec.template.spec.containers[0].env[9].valueFrom.secretKeyRef.optional diff --git a/deploy/local/build-and-deploy.sh b/deploy/local/build-and-deploy.sh index 4dce0fdc36..2251ce10a3 100755 --- a/deploy/local/build-and-deploy.sh +++ b/deploy/local/build-and-deploy.sh @@ -103,17 +103,14 @@ if [ "$helm_rc" != 0 ]; then fi # ── 4. verify 3/3 Ready ─────────────────────────────────────────────────────── -# Find the relay Deployment: everything under this release named "buzz" except -# the bundled "*-minio" Deployment. (The chart fullname collapses -# "-" to "" when the release name already contains the -# chart name, so the name isn't always "-buzz".) -DEPLOY="" -for d in $(kubectl -n "$NS" get deploy -l "app.kubernetes.io/instance=$RELEASE" \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do - case "$d" in *-minio) continue;; esac - DEPLOY="$d"; break -done -[ -n "$DEPLOY" ] || die "could not locate the relay Deployment" +# Select the relay by its explicit component label. Optional worker/sidecar +# Deployments share the release instance and must never become the rollout target. +DEPLOYMENTS=$(kubectl -n "$NS" get deploy \ + -l "app.kubernetes.io/instance=$RELEASE,app.kubernetes.io/component=relay" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') +[ "$(printf '%s\n' "$DEPLOYMENTS" | sed '/^$/d' | wc -l | tr -d ' ')" = "1" ] || \ + die "expected exactly one relay Deployment, got: ${DEPLOYMENTS:-}" +DEPLOY=$(printf '%s\n' "$DEPLOYMENTS" | sed '/^$/d') log "waiting for $REPLICAS relay pods Ready (deployment: $DEPLOY)" kubectl -n "$NS" rollout status deployment/"$DEPLOY" --timeout=4m | tee "$EVID/rollout.txt" kubectl -n "$NS" get pods -o wide | tee "$EVID/pods.txt" @@ -129,9 +126,8 @@ log "probing /_readiness on each relay pod individually" : > "$EVID/readiness.txt" FAIL=0 RELAY_PODS=$(kubectl -n "$NS" get pods \ - -l "app.kubernetes.io/name=buzz,app.kubernetes.io/instance=$RELEASE" \ - -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.app\.kubernetes\.io/component}{"\n"}{end}' \ - | awk '$2 != "minio" && $2 != "minio-init" {print $1}') + -l "app.kubernetes.io/instance=$RELEASE,app.kubernetes.io/component=relay" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') for pod in $RELAY_PODS; do body=$(kubectl -n "$NS" exec "$pod" -- \ sh -c 'curl -sS --max-time 5 http://127.0.0.1:8080/_readiness' 2>/dev/null || echo '') diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index c2e241f258..1f15a60f1e 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -23,6 +23,8 @@ CREATE TABLE community_deletion_requests ( reason TEXT, schema_manifest JSONB, storage_manifest JSONB, + destructive_storage_manifest JSONB, + destructive_storage_frozen_at TIMESTAMPTZ, inventory_manifest JSONB, inventory_digest BYTEA CHECK (inventory_digest IS NULL OR length(inventory_digest) = 32), inventory_frozen_at TIMESTAMPTZ, @@ -88,6 +90,22 @@ CREATE TABLE community_deletion_retention_exceptions ( PRIMARY KEY (request_id, exception_key) ); +CREATE TABLE community_serving_write_leases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL REFERENCES communities(id), + operation TEXT NOT NULL, + owner TEXT NOT NULL, + generation BIGINT NOT NULL DEFAULT 1 CHECK (generation > 0), + -- Community fence generation observed when this lease was acquired. + fence_generation BIGINT NOT NULL CHECK (fence_generation >= 0), + lease_until TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (community_id, id) +); +CREATE INDEX community_serving_write_leases_active + ON community_serving_write_leases (community_id, lease_until); + CREATE TABLE community_deletion_executor_heartbeats ( executor_id TEXT PRIMARY KEY, mode TEXT NOT NULL CHECK (mode IN ('run', 'drain', 'worker')), @@ -103,6 +121,7 @@ INSERT INTO _operator_global_tables (table_name, reason) VALUES ('community_deletion_approvals', 'deployment operator destructive approvals'), ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), ('community_deletion_retention_exceptions', 'deployment retention holds and expiry exceptions'), + ('community_serving_write_leases', 'deployment serving side-effect leases drained by deletion'), ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); -- Shared lock key used by both the trigger and the deletion engine. Every @@ -171,8 +190,11 @@ DECLARE expected_generation BIGINT; BEGIN IF TG_OP = 'DELETE' THEN - RAISE EXCEPTION 'community tombstones are permanent' - USING ERRCODE = 'object_not_in_prerequisite_state'; + IF OLD.deletion_state <> 'active' OR OLD.deleted_at IS NOT NULL THEN + RAISE EXCEPTION 'community tombstones are permanent' + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN OLD; END IF; expected_generation := CASE @@ -225,6 +247,7 @@ BEGIN 'community_deletion_approvals', 'community_deletion_checkpoints', 'community_deletion_retention_exceptions', + 'community_serving_write_leases', 'community_deletion_executor_heartbeats' ) LOOP diff --git a/schema/schema.sql b/schema/schema.sql index 1db7209b10..3d9f53766e 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1098,6 +1098,8 @@ CREATE TABLE community_deletion_requests ( reason TEXT, schema_manifest JSONB, storage_manifest JSONB, + destructive_storage_manifest JSONB, + destructive_storage_frozen_at TIMESTAMPTZ, inventory_manifest JSONB, inventory_digest BYTEA CHECK (inventory_digest IS NULL OR length(inventory_digest) = 32), inventory_frozen_at TIMESTAMPTZ, @@ -1159,6 +1161,22 @@ CREATE TABLE community_deletion_retention_exceptions ( created_at TIMESTAMPTZ NOT NULL DEFAULT now(), PRIMARY KEY (request_id, exception_key) ); +CREATE TABLE community_serving_write_leases ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL REFERENCES communities(id), + operation TEXT NOT NULL, + owner TEXT NOT NULL, + generation BIGINT NOT NULL DEFAULT 1 CHECK (generation > 0), + -- Community fence generation observed when this lease was acquired. + fence_generation BIGINT NOT NULL CHECK (fence_generation >= 0), + lease_until TIMESTAMPTZ NOT NULL, + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (community_id, id) +); +CREATE INDEX community_serving_write_leases_active + ON community_serving_write_leases (community_id, lease_until); + CREATE TABLE community_deletion_executor_heartbeats ( executor_id TEXT PRIMARY KEY, mode TEXT NOT NULL CHECK (mode IN ('run', 'drain', 'worker')), @@ -1173,6 +1191,7 @@ INSERT INTO _operator_global_tables (table_name, reason) VALUES ('community_deletion_approvals', 'deployment operator destructive approvals'), ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), ('community_deletion_retention_exceptions', 'deployment retention holds and expiry exceptions'), + ('community_serving_write_leases', 'deployment serving side-effect leases drained by deletion'), ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); CREATE FUNCTION community_deletion_lock_key(target UUID) RETURNS BIGINT @@ -1218,8 +1237,11 @@ DECLARE expected_generation BIGINT; BEGIN IF TG_OP = 'DELETE' THEN - RAISE EXCEPTION 'community tombstones are permanent' - USING ERRCODE = 'object_not_in_prerequisite_state'; + IF OLD.deletion_state <> 'active' OR OLD.deleted_at IS NOT NULL THEN + RAISE EXCEPTION 'community tombstones are permanent' + USING ERRCODE = 'object_not_in_prerequisite_state'; + END IF; + RETURN OLD; END IF; expected_generation := CASE WHEN NEW.deletion_fence_generation > OLD.deletion_fence_generation THEN NEW.deletion_fence_generation ELSE OLD.deletion_fence_generation END; @@ -1248,7 +1270,7 @@ BEGIN AND a.attname = 'community_id' AND NOT a.attisdropped AND c.relname NOT IN ('community_deletion_requests', 'community_deletion_approvals', 'community_deletion_checkpoints', 'community_deletion_retention_exceptions', - 'community_deletion_executor_heartbeats') + 'community_serving_write_leases', 'community_deletion_executor_heartbeats') LOOP EXECUTE format('CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %I ' 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', From 4dff6d9932ebd58a078899fab1c70073a7f56c5a Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Fri, 31 Jul 2026 15:55:28 -0700 Subject: [PATCH 03/51] test: cover deletion lifecycle safety boundaries Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index d9b680c2cf..f69d46d96f 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -1920,6 +1920,18 @@ mod postgres_tests { .expect("claim before approval") .is_none()); + // Approval binds structural/storage taxonomy, not live row counts. A + // serving write between inventory and approval must not make an active + // community undeletable. + db.add_to_allowlist(request.community_id, &[7_u8; 32], &[8_u8; 32], None) + .await + .expect("post-inventory serving write"); + let current_schema = store + .inventory_schema(request.community_id) + .await + .expect("live schema after row churn"); + assert_eq!(current_schema, inventory.schema); + let approved = store .approve(request.id, "approver-a", Some("reviewed")) .await @@ -2088,6 +2100,20 @@ mod postgres_tests { .freeze_destructive_storage_manifest(&token, &inventory.storage) .await .expect("freeze destructive storage"); + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await + .expect("identical destructive manifest retry"); + let mut drifted_storage = inventory.storage.clone(); + drifted_storage + .media_upload_keys + .push("media/drifted-after-fence".to_string()); + assert!(matches!( + store + .freeze_destructive_storage_manifest(&token, &drifted_storage) + .await, + Err(DbError::DeletionSafety(_)) + )); store.mark_drained(&token).await.expect("drain"); store .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) @@ -2136,5 +2162,13 @@ mod postgres_tests { .await .expect("serving lookup") .is_none()); + let direct_delete = sqlx::query("DELETE FROM communities WHERE id = $1") + .bind(request.community_id.as_uuid()) + .execute(&db.pool) + .await + .expect_err("tombstone row must be permanent"); + assert!(direct_delete + .to_string() + .contains("tombstones are permanent")); } } From af4c4a6637c9491090917a51c2935d6fa0c2b073 Mon Sep 17 00:00:00 2001 From: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Date: Fri, 31 Jul 2026 16:10:32 -0700 Subject: [PATCH 04/51] fix: harden deletion executor lease handling Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> --- crates/buzz-deletion/src/lib.rs | 375 +++++++++++++++++++++++++++++--- 1 file changed, 340 insertions(+), 35 deletions(-) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 0def7a7db9..a77e00d7df 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -37,6 +37,14 @@ fn heartbeat_interval() -> Duration { HEARTBEAT_INTERVAL } +fn worker_health_stale_after() -> u64 { + HEARTBEAT_INTERVAL.as_secs().saturating_mul(3) +} + +#[derive(Debug, Clone, thiserror::Error)] +#[error("deletion execution lease heartbeat failed")] +struct DeletionLeaseLost; + #[derive(Debug, Clone, thiserror::Error)] #[error("{message}")] struct ServingWriteLeaseLost { @@ -86,13 +94,13 @@ impl ServingWriteGuard { self.verify().await?; let output = tokio::select! { biased; + output = operation => output, _ = self.lost.cancelled() => { return Err(ServingWriteLeaseLost { message: "serving write lease heartbeat was lost".to_string(), } .into()) } - output = operation => output, }; self.verify().await?; Ok(output) @@ -340,7 +348,8 @@ impl WorkerHealth { .as_secs(); !self.draining.load(Ordering::Relaxed) && self.dependencies_ready.load(Ordering::Relaxed) - && now.saturating_sub(self.last_heartbeat_epoch.load(Ordering::Relaxed)) <= 30 + && now.saturating_sub(self.last_heartbeat_epoch.load(Ordering::Relaxed)) + <= worker_health_stale_after() } } @@ -623,6 +632,24 @@ async fn run_loop( } } +async fn stop_claim_executor( + services: &Services, + mode: LoopMode, + token: &LeaseToken, +) -> Result<()> { + // A failed draining heartbeat must not prevent the generation-checked release + // attempt. `stop_executor` cannot clear a successor's reclaimed lease. + let _ = services + .store + .heartbeat(token, mode.as_str(), DEFAULT_LEASE_DURATION, true) + .await; + services + .store + .stop_executor(Some(token), &token.owner) + .await?; + Ok(()) +} + async fn execute_claim( services: &Services, mode: LoopMode, @@ -633,14 +660,7 @@ async fn execute_claim( let token = claim.lease.clone(); loop { if shutdown.is_cancelled() { - services - .store - .heartbeat(&token, mode.as_str(), DEFAULT_LEASE_DURATION, true) - .await?; - services - .store - .stop_executor(Some(&token), &token.owner) - .await?; + stop_claim_executor(services, mode, &token).await?; let request = services.store.get(token.request_id).await?; return Ok(run_output(request)); } @@ -653,8 +673,19 @@ async fn execute_claim( let stage_result = run_stage_with_heartbeat(services, mode, &claim, shutdown, Arc::clone(&health)).await; match stage_result { - Ok(()) => {} - Err(error) => { + StageOutcome::Completed => {} + StageOutcome::Shutdown => { + stop_claim_executor(services, mode, &token).await?; + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); + } + StageOutcome::Failed(error) => { + let request = services.store.get(token.request_id).await?; + if request.lease_owner.as_deref() != Some(&token.owner) + || request.lease_generation != token.generation + { + return Ok(run_output(request)); + } let message = format!("{error:#}"); if is_permanent_error(&error) { services @@ -699,13 +730,38 @@ async fn dependencies_ready(services: &Services) -> bool { redis_ok && storage_ok } +enum StageOutcome { + Completed, + Shutdown, + Failed(anyhow::Error), +} + +async fn await_stage( + stage: F, + shutdown: &CancellationToken, + heartbeat_error: &CancellationToken, +) -> StageOutcome +where + F: std::future::Future>, +{ + tokio::select! { + biased; + _ = shutdown.cancelled() => StageOutcome::Shutdown, + _ = heartbeat_error.cancelled() => StageOutcome::Failed(DeletionLeaseLost.into()), + result = stage => match result { + Ok(()) => StageOutcome::Completed, + Err(error) => StageOutcome::Failed(error), + }, + } +} + async fn run_stage_with_heartbeat( services: &Services, mode: LoopMode, claim: &ClaimedDeletion, shutdown: &CancellationToken, health: Arc, -) -> Result<()> { +) -> StageOutcome { let heartbeat_services = services.clone(); let heartbeat_token = claim.lease.clone(); let heartbeat_mode = mode.as_str(); @@ -742,22 +798,26 @@ async fn run_stage_with_heartbeat( } }); - let stage = tokio::select! { - _ = shutdown.cancelled() => Err(anyhow::anyhow!("executor shutdown requested")), - _ = heartbeat_error.cancelled() => Err(anyhow::anyhow!("deletion lease heartbeat failed")), - result = execute_stage(services, claim) => result, - }; + let stage = await_stage( + execute_stage(services, claim, &heartbeat_error), + shutdown, + &heartbeat_error, + ) + .await; heartbeat_shutdown.cancel(); match heartbeat.await { Ok(()) => stage, - Err(error) => Err(anyhow::anyhow!("deletion heartbeat task failed: {error}")), + Err(error) => { + StageOutcome::Failed(anyhow::anyhow!("deletion heartbeat task failed: {error}")) + } } } -async fn guarded_external_step( +async fn run_guarded_external_step( services: &Services, token: &LeaseToken, stage: DeletionStage, + heartbeat_lost: &CancellationToken, operation: F, ) -> Result<()> where @@ -765,12 +825,23 @@ where Fut: std::future::Future>, { services.store.verify_execution_token(token, stage).await?; - operation().await?; + let result = tokio::select! { + biased; + _ = heartbeat_lost.cancelled() => { + return Err(DeletionLeaseLost.into()); + } + result = operation() => result, + }; + result?; services.store.verify_execution_token(token, stage).await?; Ok(()) } -async fn execute_stage(services: &Services, claim: &ClaimedDeletion) -> Result<()> { +async fn execute_stage( + services: &Services, + claim: &ClaimedDeletion, + heartbeat_lost: &CancellationToken, +) -> Result<()> { let request = &claim.request; let token = token_with_current_fence(&claim.lease, request); match request.stage { @@ -803,7 +874,20 @@ async fn execute_stage(services: &Services, claim: &ClaimedDeletion) -> Result<( services.store.fence(&token).await?; } DeletionStage::Fenced => { - publish_disconnect_community(&services.redis, request.community_id).await?; + services + .store + .verify_execution_token(&token, DeletionStage::Fenced) + .await?; + let disconnect = tokio::select! { + biased; + _ = heartbeat_lost.cancelled() => Err(DeletionLeaseLost.into()), + result = publish_disconnect_community(&services.redis, request.community_id) => result, + }; + disconnect?; + services + .store + .verify_execution_token(&token, DeletionStage::Fenced) + .await?; let destructive = match request.destructive_storage_manifest.clone() { Some(value) => serde_json::from_value(value)?, None => { @@ -858,19 +942,31 @@ async fn execute_stage(services: &Services, claim: &ClaimedDeletion) -> Result<( } } for key in &storage.tenant_keys { - guarded_external_step(services, &token, DeletionStage::Drained, || async { - services.media.delete(key).await?; - Ok(()) - }) + run_guarded_external_step( + services, + &token, + DeletionStage::Drained, + heartbeat_lost, + || async { + services.media.delete(key).await?; + Ok(()) + }, + ) .await?; - guarded_external_step(services, &token, DeletionStage::Drained, || async { - if services.media.head(key).await? { - return Err(transient(format!( - "object binding still exists after delete: {key}" - ))); - } - Ok(()) - }) + run_guarded_external_step( + services, + &token, + DeletionStage::Drained, + heartbeat_lost, + || async { + if services.media.head(key).await? { + return Err(transient(format!( + "object binding still exists after delete: {key}" + ))); + } + Ok(()) + }, + ) .await?; } services @@ -1154,6 +1250,82 @@ fn print_json(value: &impl Serialize) -> Result<()> { mod tests { use super::*; + async fn claimed_test_deletion(prefix: &str) -> Option<(Services, ClaimedDeletion)> { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .ok()?; + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect deletion engine test DB"); + let db = Db::from_pool(pool); + db.migrate().await.expect("migrate deletion engine test DB"); + let store = db.deletion_store(); + let host = format!("{prefix}-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("create deletion engine test community"); + let request = store + .submit(&host, "test", None) + .await + .expect("submit deletion request"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community.id) + .await + .expect("inventory schema"), + storage: StorageManifest { + version: 1, + tenant_keys: Vec::new(), + git_pointer_keys: Vec::new(), + media_sidecar_keys: Vec::new(), + media_upload_keys: Vec::new(), + retained_shared_cas_keys: Vec::new(), + unknown_keys: Vec::new(), + unsupported_version_keys: Vec::new(), + }, + }; + store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze deletion inventory"); + store + .approve(request.id, "test", None) + .await + .expect("approve deletion request"); + let claim = store + .claim_specific(request.id, "test-executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim deletion request") + .expect("runnable deletion request"); + let services = Services { + store, + media: Arc::new( + MediaStorage::new(&buzz_media::MediaConfig { + s3_endpoint: "http://127.0.0.1:1".to_string(), + s3_access_key: "unused".to_string(), + s3_secret_key: "unused".to_string(), + s3_bucket: "unused".to_string(), + s3_region: "us-east-1".to_string(), + s3_addressing_style: buzz_media::S3AddressingStyle::Path, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }) + .expect("construct unused media service"), + ), + redis: deadpool_redis::Config::from_url("redis://127.0.0.1:1") + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("construct unused Redis pool"), + }; + Some((services, claim)) + } + #[test] fn permanent_failures_are_typed_not_string_classified() { let permanent_error = permanent("catalog drift"); @@ -1170,6 +1342,34 @@ mod tests { assert!(!is_permanent_error(&db_transient)); } + #[test] + fn worker_readiness_requires_dependencies_heartbeat_and_not_draining() { + let health = WorkerHealth::default(); + assert!(!health.ready()); + health.dependencies_ready.store(true, Ordering::Relaxed); + health.mark_heartbeat(); + assert!(health.ready()); + health.draining.store(true, Ordering::Relaxed); + assert!(!health.ready()); + health.draining.store(false, Ordering::Relaxed); + health.last_heartbeat_epoch.store(1, Ordering::Relaxed); + assert!(!health.ready()); + } + + #[test] + fn worker_configuration_requires_every_destructive_dependency() { + let variable = format!("BUZZ_DELETION_REQUIRED_TEST_{}", Uuid::new_v4().simple()); + assert!(required_env(&variable).is_err()); + std::env::set_var(&variable, " "); + assert!(required_env(&variable).is_err()); + std::env::set_var(&variable, "configured"); + assert_eq!( + required_env(&variable).expect("configured environment variable"), + "configured" + ); + std::env::remove_var(&variable); + } + #[test] fn redis_absence_requires_terminal_cursor_and_all_pages_empty() { assert!(!scan_proves_absence(&[(9, Vec::new())])); @@ -1226,6 +1426,111 @@ mod tests { ); } + #[tokio::test] + async fn guarded_external_step_rejects_preexisting_heartbeat_loss_without_polling_operation() { + let Some((services, claim)) = claimed_test_deletion("deletion-heartbeat").await else { + return; + }; + let heartbeat_lost = CancellationToken::new(); + heartbeat_lost.cancel(); + let polled = Arc::new(AtomicBool::new(false)); + let operation_polled = Arc::clone(&polled); + let result = run_guarded_external_step( + &services, + &claim.lease, + DeletionStage::Approved, + &heartbeat_lost, + || async move { + operation_polled.store(true, Ordering::Relaxed); + Ok(()) + }, + ) + .await; + assert!(result.is_err(), "heartbeat loss must abort the side effect"); + assert!( + result + .expect_err("heartbeat loss error") + .downcast_ref::() + .is_some(), + "heartbeat loss must stay typed" + ); + assert!( + !polled.load(Ordering::Relaxed), + "a pre-cancelled heartbeat must win before polling the operation" + ); + } + + #[tokio::test] + async fn shutdown_during_stage_releases_claim_without_recording_retry() { + let Some((services, claim)) = claimed_test_deletion("deletion-shutdown").await else { + return; + }; + let request_id = claim.request.id; + let retry_count = claim.request.retry_count; + let shutdown = CancellationToken::new(); + let cancel = shutdown.clone(); + let services_for_run = services.clone(); + let executor = tokio::spawn(async move { + execute_claim( + &services_for_run, + LoopMode::Worker, + claim, + &shutdown, + Arc::new(WorkerHealth::default()), + ) + .await + }); + tokio::time::sleep(Duration::from_millis(10)).await; + cancel.cancel(); + let output = tokio::time::timeout(Duration::from_secs(2), executor) + .await + .expect("shutdown must cancel the active stage") + .expect("deletion executor task") + .expect("graceful deletion executor shutdown"); + let request = services + .store + .get(request_id) + .await + .expect("load deletion request after shutdown"); + assert_eq!(output.stage, DeletionStage::Approved); + assert_eq!(request.stage, DeletionStage::Approved); + assert_eq!(request.retry_count, retry_count); + assert!(request.last_error.is_none()); + assert!(request.lease_owner.is_none()); + assert!(request.lease_until.is_none()); + } + + #[tokio::test] + async fn stage_wait_treats_shutdown_as_control_flow() { + let shutdown = CancellationToken::new(); + shutdown.cancel(); + let heartbeat_lost = CancellationToken::new(); + let outcome = await_stage( + std::future::pending::>(), + &shutdown, + &heartbeat_lost, + ) + .await; + assert!(matches!(outcome, StageOutcome::Shutdown)); + } + + #[tokio::test] + async fn stage_wait_prioritizes_heartbeat_loss_over_a_ready_operation() { + let shutdown = CancellationToken::new(); + let heartbeat_lost = CancellationToken::new(); + heartbeat_lost.cancel(); + let outcome = await_stage(async { Ok(()) }, &shutdown, &heartbeat_lost).await; + match outcome { + StageOutcome::Failed(error) => assert!( + error.downcast_ref::().is_some(), + "heartbeat loss must stay typed" + ), + StageOutcome::Completed | StageOutcome::Shutdown => { + panic!("preexisting heartbeat loss must win") + } + } + } + #[tokio::test] async fn shutdown_token_can_interrupt_idle_worker_sleep() { let token = CancellationToken::new(); From dc6e098b6a5ca73faf8baf89e6cb71c3600f4a86 Mon Sep 17 00:00:00 2001 From: Kalvin Chau Date: Fri, 31 Jul 2026 17:08:02 -0700 Subject: [PATCH 05/51] fix(relay): restore community write fence on persistent ingest the checkpoint removed the pre-lease fence assert from ingest_event_inner without a replacement, leaving the shared ws/http persistent-event seam unfenced during community deletion. restore it as a fail-closed is_serving_active pre-check: inactive communities reject with the same restricted: wire text as the ephemeral path, and a fence-lookup outage maps to internal/500 instead of admitting the write or blaming the client. regressions: unit tests pin the three mapping arms, and an ignored postgres test drives the real deletion lifecycle (submit, freeze inventory, approve, claim, fence) and proves the exact predicate ingest consults flips from admit to refuse. Signed-off-by: Kalvin Chau Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> --- crates/buzz-relay/src/handlers/ingest.rs | 139 +++++++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index cd9f20b5f4..e545214917 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -232,6 +232,24 @@ pub enum IngestError { Internal(String), } +/// Map the durable community write-fence lookup onto the ingest error taxonomy. +/// +/// An inactive community is an authorization decision and keeps the exact +/// `restricted:` wire text the ephemeral path uses. A lookup outage is a +/// server fault and fails closed as `error:`/500 — a Postgres blip can +/// neither admit a write past the fence nor read as a client mistake. +fn map_serving_fence_state(active: Result) -> Result<(), IngestError> { + match active { + Ok(true) => Ok(()), + Ok(false) => Err(IngestError::Rejected( + "restricted: community writes are fenced".into(), + )), + Err(error) => Err(IngestError::Internal(format!( + "error: checking community write fence: {error}" + ))), + } +} + fn map_relay_admin_error(error: super::relay_admin::RelayAdminError) -> IngestError { use super::relay_admin::RelayAdminError; match error { @@ -1863,6 +1881,17 @@ async fn ingest_event_inner( let kind_u32 = event_kind_u32(&event); debug!(event_id = %event_id_hex, kind = kind_u32, "ingest_event"); + // Durable community write fence: persistent ingest is a DB write the + // deletion engine cannot exclude via serving-write leases (those cover + // external side effects only), so the shared WS/HTTP seam must refuse + // writes once the community leaves the active lifecycle state. Row churn + // inside the remaining race window is swept by the destructive DB stage. + map_serving_fence_state( + buzz_deletion::store(&state.db) + .is_serving_active(tenant.community()) + .await, + )?; + if kind_u32 == KIND_AUTH { return Err(IngestError::Rejected( "invalid: AUTH events cannot be submitted".into(), @@ -3093,6 +3122,116 @@ mod tests { } } + /// An active community passes the durable write fence untouched. + #[test] + fn serving_fence_active_community_admits_write() { + assert!(map_serving_fence_state(Ok(true)).is_ok()); + } + + /// A fenced/tombstoned/archived community is an authorization decision: + /// `restricted:` and (via `bridge.rs`) HTTP 400 — with the exact wire text + /// the ephemeral WS path uses, so clients see one refusal vocabulary. + #[test] + fn serving_fence_inactive_community_maps_to_restricted() { + match map_serving_fence_state(Ok(false)) { + Err(IngestError::Rejected(msg)) => { + assert_eq!(msg, "restricted: community writes are fenced"); + } + other => panic!("fenced community must map to Rejected, got {other:?}"), + } + } + + /// A fence-lookup outage is a server fault and must fail closed as + /// `error:`/500 — a Postgres blip can neither admit a write past the + /// fence nor be reported to an innocent client as a bad request. + #[test] + fn serving_fence_lookup_outage_fails_closed_as_internal() { + let outage = buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut); + match map_serving_fence_state(Err(outage)) { + Err(IngestError::Internal(msg)) => { + assert!( + msg.starts_with("error: "), + "fence outages need the `error:` NIP-01 prefix, got {msg:?}" + ); + } + other => panic!("fence lookup failure must map to Internal, got {other:?}"), + } + } + + /// Production-path regression: the exact predicate `ingest_event_inner` + /// consults must admit writes while a community is active and refuse them + /// once the community deletion lifecycle fences it. + #[tokio::test] + #[ignore = "requires Postgres"] + async fn ingest_write_fence_follows_community_deletion_lifecycle() { + use buzz_db::deletion::{FrozenInventory, StorageManifest, DEFAULT_LEASE_DURATION}; + + let url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); // sadscan:disable np.postgres.1 + let pool = sqlx::PgPool::connect(&url).await.expect("connect test DB"); + let db = buzz_db::Db::from_pool(pool); + let store = buzz_deletion::store(&db); + + let host = format!("lane3-fence-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("community") + .id; + + assert!( + map_serving_fence_state(store.is_serving_active(community).await).is_ok(), + "active community must admit persistent ingest" + ); + + let submitted = store + .submit( + &host, + "test-operator", + Some("lane3 ingest fence regression"), + ) + .await + .expect("submit"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("schema inventory"), + storage: StorageManifest { + version: 1, + tenant_keys: Vec::new(), + git_pointer_keys: Vec::new(), + media_sidecar_keys: Vec::new(), + media_upload_keys: Vec::new(), + retained_shared_cas_keys: Vec::new(), + unknown_keys: Vec::new(), + unsupported_version_keys: Vec::new(), + }, + }; + let request = store + .freeze_inventory(submitted.id, &inventory) + .await + .expect("freeze inventory"); + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.fence(&claim.lease).await.expect("fence"); + + match map_serving_fence_state(store.is_serving_active(community).await) { + Err(IngestError::Rejected(msg)) => { + assert_eq!(msg, "restricted: community writes are fenced"); + } + other => panic!("fenced community must refuse persistent ingest, got {other:?}"), + } + } + #[derive(Debug, Default)] struct VecTracer { steps: Mutex>, From d918d47821f38931d9ae3ee2f288deebbaf0627f Mon Sep 17 00:00:00 2001 From: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Date: Fri, 31 Jul 2026 19:12:16 -0700 Subject: [PATCH 06/51] fix: close community deletion serving-write races Order fencing behind admitted serving-write leases, keep external write heartbeats live through durable finalization, and make Git publication hold the community lock through the relay event commit. Harden tenant trigger coverage for cross-community updates and future tables, then fail startup and readiness closed on live catalog drift. Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 240 ++++++++++--- crates/buzz-db/src/error.rs | 14 + crates/buzz-db/src/event.rs | 2 +- crates/buzz-db/src/lib.rs | 48 +++ crates/buzz-db/src/migration.rs | 299 ++++++++++++++++ crates/buzz-deletion/src/lib.rs | 23 +- crates/buzz-relay/src/api/git/transport.rs | 390 +++++++++++++++++++-- crates/buzz-relay/src/main.rs | 6 + crates/buzz-relay/src/push_runtime.rs | 1 - crates/buzz-relay/src/router.rs | 26 +- migrations/0028_community_deletion.sql | 126 +++++-- schema/schema.sql | 140 ++++++-- 12 files changed, 1179 insertions(+), 136 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index f69d46d96f..71070874e8 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -581,8 +581,12 @@ impl DeletionStore { }) } - /// Build and validate a live PostgreSQL schema inventory. - pub async fn inventory_schema(&self, community: CommunityId) -> Result { + /// Validate the live scoped-table and write-fence catalog before serving. + /// + /// This uses the same exact-set checks as deletion inventory but does not + /// require a community row. Schema drift therefore fails startup/readiness + /// even before an operator submits a deletion request. + pub async fn validate_catalog(&self) -> Result<()> { let migration_version: i64 = sqlx::query_scalar( "SELECT COALESCE(max(version), 0) FROM _sqlx_migrations WHERE success", ) @@ -594,12 +598,12 @@ impl DeletionStore { ))); } - let live_tables = self.live_scoped_tables().await?; let expected = EXPECTED_SCOPED_TABLES .iter() .copied() .map(str::to_owned) .collect::>(); + let live_tables = self.live_scoped_tables().await?; if live_tables != expected { let missing = expected .difference(&live_tables) @@ -622,16 +626,28 @@ impl DeletionStore { .difference(&fenced_tables) .cloned() .collect::>(); + let unknown = fenced_tables + .difference(&expected) + .cloned() + .collect::>(); return Err(DbError::DeletionSafety(format!( - "community deletion write-fence drift (missing={})", - missing.join(",") + "community deletion write-fence drift (missing={}, unknown={})", + missing.join(","), + unknown.join(",") ))); } + Ok(()) + } + /// Build and validate a live PostgreSQL schema inventory. + pub async fn inventory_schema(&self, community: CommunityId) -> Result { + self.validate_catalog().await?; + let live_tables = self.live_scoped_tables().await?; + let fenced_tables = self.live_fenced_tables().await?; let _ = community; // counts are intentionally not approval-bound for a live tenant. Ok(SchemaManifest { revision: CATALOG_REVISION, - migration_version, + migration_version: EXPECTED_MIGRATION_VERSION, scoped_tables: live_tables.into_iter().collect(), row_counts: BTreeMap::new(), fenced_tables: fenced_tables.into_iter().collect(), @@ -882,6 +898,23 @@ impl DeletionStore { .bind(token.community_id.as_uuid()) .execute(&mut *tx) .await?; + let active_serving_writes = sqlx::query( + "SELECT count(*)::BIGINT AS active_count, \ + COALESCE(array_agg(DISTINCT operation ORDER BY operation), ARRAY[]::TEXT[]) AS operations \ + FROM community_serving_write_leases \ + WHERE community_id = $1 AND lease_until >= now()", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + let active_count: i64 = active_serving_writes.try_get("active_count")?; + if active_count > 0 { + return Err(DbError::ServingWritesNotDrained { + community_id: *token.community_id.as_uuid(), + active_count, + operations: active_serving_writes.try_get("operations")?, + }); + } let current_generation: i64 = sqlx::query_scalar( "SELECT deletion_fence_generation FROM communities WHERE id = $1 FOR UPDATE", ) @@ -1353,6 +1386,11 @@ impl DeletionStore { lease_duration: Duration, ) -> Result<()> { let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut *tx) + .await?; let lease_until: Option> = sqlx::query_scalar( "UPDATE community_serving_write_leases lease \ SET lease_until = now() + make_interval(secs => $6), heartbeat_at = now() \ @@ -1361,10 +1399,9 @@ impl DeletionStore { AND lease.generation = $4 AND lease.fence_generation = $5 \ AND lease.lease_until >= now() \ AND community.id = lease.community_id \ - AND ((community.deletion_state = 'active' \ - AND community.deletion_fence_generation = lease.fence_generation) \ - OR (community.deletion_state = 'fenced' \ - AND community.deletion_fence_generation = lease.fence_generation + 1)) \ + AND community.deletion_state = 'active' \ + AND community.deleted_at IS NULL \ + AND community.deletion_fence_generation = lease.fence_generation \ RETURNING lease.lease_until", ) .bind(lease.id) @@ -1373,11 +1410,13 @@ impl DeletionStore { .bind(lease.generation) .bind(lease.fence_generation) .bind(lease_seconds) - .fetch_optional(&self.pool) + .fetch_optional(&mut *tx) .await?; - lease.lease_until = lease_until.ok_or_else(|| { + let lease_until = lease_until.ok_or_else(|| { DbError::AccessDenied(format!("stale serving write lease {}", lease.id)) })?; + tx.commit().await?; + lease.lease_until = lease_until; Ok(()) } @@ -1402,6 +1441,11 @@ impl DeletionStore { /// Check that an external side-effect lease and the community's active /// lifecycle are still current. pub async fn verify_serving_write_lease(&self, lease: &ServingWriteLease) -> Result<()> { + let mut tx = self.pool.begin().await?; + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut *tx) + .await?; let valid: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ JOIN communities community ON community.id = lease.community_id \ @@ -1409,19 +1453,18 @@ impl DeletionStore { AND lease.generation = $4 AND lease.fence_generation = $5 \ AND lease.lease_until >= now() \ AND community.deleted_at IS NULL \ - AND ((community.deletion_state = 'active' \ - AND community.deletion_fence_generation = lease.fence_generation) \ - OR (community.deletion_state = 'fenced' \ - AND community.deletion_fence_generation = lease.fence_generation + 1)))", + AND community.deletion_state = 'active' \ + AND community.deletion_fence_generation = lease.fence_generation)", ) .bind(lease.id) .bind(lease.community_id.as_uuid()) .bind(&lease.owner) .bind(lease.generation) .bind(lease.fence_generation) - .fetch_one(&self.pool) + .fetch_one(&mut *tx) .await?; if valid { + tx.commit().await?; Ok(()) } else { Err(DbError::AccessDenied(format!( @@ -1472,14 +1515,7 @@ impl DeletionStore { AND NOT c.relispartition AND a.attname = 'community_id' AND NOT a.attisdropped - AND c.relname NOT IN ( - 'community_deletion_requests', - 'community_deletion_approvals', - 'community_deletion_checkpoints', - 'community_deletion_retention_exceptions', - 'community_serving_write_leases', - 'community_deletion_executor_heartbeats' - ) + AND NOT community_write_fence_excluded_table(c.relname) ORDER BY c.relname "#, ) @@ -1500,6 +1536,12 @@ impl DeletionStore { AND NOT trigger.tgisinternal AND NOT c.relispartition AND procedure.proname = 'enforce_community_write_fence' + AND trigger.tgenabled = 'O' + AND (trigger.tgtype & 1) = 1 + AND (trigger.tgtype & 2) = 2 + AND (trigger.tgtype & 4) = 4 + AND (trigger.tgtype & 8) = 8 + AND (trigger.tgtype & 16) = 16 ORDER BY c.relname "#, ) @@ -2028,7 +2070,7 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn external_serving_lease_blocks_drain_without_holding_a_pool_connection() { + async fn active_serving_lease_retries_fence_without_lifecycle_change() { let (db, store) = store().await; let (request, _) = inventoried_request(&db, &store).await; store @@ -2040,7 +2082,7 @@ mod postgres_tests { .await .expect("claim") .expect("won claim"); - let mut serving = store + let serving = store .acquire_serving_write_lease( request.community_id, "test_external", @@ -2049,31 +2091,147 @@ mod postgres_tests { ) .await .expect("serving lease"); - let generation = store.fence(&claim.lease).await.expect("fence"); + + let error = store + .fence(&claim.lease) + .await + .expect_err("active serving lease must make fence retry"); + assert!(matches!( + error, + DbError::ServingWritesNotDrained { + active_count: 1, + ref operations, + .. + } if operations == &["test_external"] + )); + let unchanged = store.get(request.id).await.expect("request after retry"); + assert_eq!(unchanged.stage, DeletionStage::Approved); + assert_eq!(unchanged.fence_generation, None); + assert!(store + .is_serving_active(request.community_id) + .await + .expect("community remains active")); + + assert!(store + .release_serving_write_lease(&serving) + .await + .expect("release")); + let generation = store.fence(&claim.lease).await.expect("retry fence"); + assert_eq!(generation, 1); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_lease_acquisition_and_fence_have_exactly_one_winner() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; store - .renew_serving_write_lease(&mut serving, DEFAULT_LEASE_DURATION) + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + + let mut lock_holder = db.pool.acquire().await.expect("lock connection"); + sqlx::query("BEGIN") + .execute(&mut *lock_holder) .await - .expect("pre-fence side effect continues heartbeating while fenced"); + .expect("begin lock transaction"); + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(request.community_id.as_uuid()) + .execute(&mut *lock_holder) + .await + .expect("hold ordering lock"); + + let acquire_store = store.clone(); + let community = request.community_id; + let acquire = tokio::spawn(async move { + acquire_store + .acquire_serving_write_lease( + community, + "race_acquire", + "race-owner", + DEFAULT_LEASE_DURATION, + ) + .await + }); + let fence_store = store.clone(); + let fence_token = claim.lease.clone(); + let fence = tokio::spawn(async move { fence_store.fence(&fence_token).await }); + sqlx::query("COMMIT") + .execute(&mut *lock_holder) + .await + .expect("release ordering lock"); + + let lease_result = acquire.await.expect("acquire task"); + let fence_result = fence.await.expect("fence task"); + match (lease_result, fence_result) { + (Ok(lease), Err(DbError::ServingWritesNotDrained { .. })) => { + assert!(store + .is_serving_active(request.community_id) + .await + .expect("active after lease wins")); + assert!(store + .release_serving_write_lease(&lease) + .await + .expect("release winner")); + } + (Err(DbError::AccessDenied(_)), Ok(_)) => { + assert!(!store + .is_serving_active(request.community_id) + .await + .expect("fenced after deletion wins")); + } + other => panic!("invalid acquisition/fence race outcome: {other:?}"), + } + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn external_serving_lease_blocks_drain_without_holding_a_pool_connection() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; store - .verify_serving_write_lease(&serving) + .approve(request.id, "approver", None) .await - .expect("pre-fence side effect remains valid while draining"); - let token = LeaseToken { - fence_generation: Some(generation), - ..claim.lease - }; - assert!( - store.mark_drained(&token).await.is_err(), - "fenced deletion must wait for a pre-fence external effect lease" - ); + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + let serving = store + .acquire_serving_write_lease( + request.community_id, + "test_external", + "test-owner", + DEFAULT_LEASE_DURATION, + ) + .await + .expect("serving lease"); + assert!(matches!( + store.fence(&claim.lease).await, + Err(DbError::ServingWritesNotDrained { .. }) + )); assert!(store .release_serving_write_lease(&serving) .await .expect("release")); + let generation = store + .fence(&claim.lease) + .await + .expect("fence after release"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; store .mark_drained(&token) .await - .expect("drain after release"); + .expect("drain immediately after successful fence"); } #[tokio::test] diff --git a/crates/buzz-db/src/error.rs b/crates/buzz-db/src/error.rs index 21e7e40ed6..593eea1cca 100644 --- a/crates/buzz-db/src/error.rs +++ b/crates/buzz-db/src/error.rs @@ -45,6 +45,20 @@ pub enum DbError { #[error("invalid data: {0}")] InvalidData(String), + /// A serving write admitted before the lifecycle transition is still live. + /// This is an ordinary retryable drain condition, not a safety violation. + #[error( + "community {community_id} still has {active_count} active serving write lease(s): {operations:?}" + )] + ServingWritesNotDrained { + /// Community whose lifecycle transition must retry. + community_id: uuid::Uuid, + /// Number of currently unexpired serving-write leases. + active_count: i64, + /// Distinct operation categories holding those leases. + operations: Vec, + }, + /// A deletion safety invariant is structurally violated and requires /// operator/code remediation rather than blind retry. #[error("deletion safety error: {0}")] diff --git a/crates/buzz-db/src/event.rs b/crates/buzz-db/src/event.rs index e1b45aa3a1..db15057171 100644 --- a/crates/buzz-db/src/event.rs +++ b/crates/buzz-db/src/event.rs @@ -1111,7 +1111,7 @@ pub struct ThreadMetadataParams<'a> { pub broadcast: bool, } -async fn insert_event_with_thread_metadata_tx( +pub(crate) async fn insert_event_with_thread_metadata_tx( tx: &mut Transaction<'_, Postgres>, community_id: CommunityId, event: &Event, diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 3e533e26bd..cdb01b6c8c 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -1020,6 +1020,11 @@ impl Db { sqlx::query("SELECT 1").execute(&self.pool).await.is_ok() } + /// Validate the live community-deletion tenant catalog and write fences. + pub async fn validate_deletion_catalog(&self) -> Result<()> { + self.deletion_store().validate_catalog().await + } + /// Returns pool utilisation stats for metrics emission. /// /// `size` — total connections (idle + active) @@ -1654,6 +1659,49 @@ impl Db { Ok(result) } + /// Insert an event while holding the admitted serving-write lease's + /// community ordering lock through commit. + /// + /// External side effects use a durable lease rather than one long-lived DB + /// transaction. Their final database mutation must still share the + /// acquire/fence advisory-lock order so a fence cannot commit between the + /// caller's lease verification and this insert. + pub async fn insert_event_with_serving_write_guard( + &self, + community_id: CommunityId, + event: &nostr::Event, + channel_id: Option, + ) -> Result<(StoredEvent, bool)> { + let kind_u16 = event.kind.as_u16(); + let kind_u32 = u32::from(kind_u16); + if kind_u32 == buzz_core::kind::KIND_AUTH { + return Err(DbError::AuthEventRejected); + } + if buzz_core::kind::is_ephemeral(kind_u32) { + return Err(DbError::EphemeralEventRejected(kind_u16)); + } + + let mut tx = self.pool.begin().await?; + self.deletion_store() + .guard_transaction(&mut tx, community_id) + .await?; + let result = event::insert_event_with_thread_metadata_tx( + &mut tx, + community_id, + event, + channel_id, + None, + ) + .await?; + tx.commit().await?; + if result.1 { + if let Err(e) = insert_mentions(&self.pool, community_id, event, channel_id).await { + tracing::warn!(event_id = %event.id, "Failed to insert mentions: {e}"); + } + } + Ok(result) + } + /// Queries events matching the given filter parameters. /// /// Always reads from the WRITER pool. If the result influences a write diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 81fcaf6b5a..6aa481d55a 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -495,6 +495,106 @@ mod tests { .collect() } + fn migrations_missing_community_write_fence_attachment() -> Vec { + let mut migrations: Vec<_> = MIGRATOR.iter().collect(); + migrations.sort_by_key(|migration| migration.version); + migrations + .into_iter() + .filter(|migration| migration.version > 27) + .flat_map(|migration| { + let statements = split_sql_statements(migration.sql.as_str()); + let attachments = statements + .iter() + .filter_map(|statement| { + let normalized = normalize_sql(statement); + if !normalized.contains("attach_community_write_fence(") { + return None; + } + let call = normalized.split_once("attach_community_write_fence(")?.1; + let table = call + .split(')') + .next()? + .trim() + .split("::") + .next()? + .trim_matches(|ch| ch == '\'' || ch == '"') + .rsplit('.') + .next()? + .to_owned(); + (!table.is_empty()).then_some(table) + }) + .collect::>(); + statements.into_iter().filter_map(move |statement| { + let normalized = normalize_sql(&statement); + let table = if normalized.starts_with("create table") + && normalized.contains("community_id") + && !normalized.contains(" partition of ") + { + identifier_after_keyword(&statement, "create table") + } else if normalized.starts_with("alter table") + && normalized.contains("add") + && normalized.contains("community_id") + { + identifier_after_keyword(&statement, "alter table") + } else { + None + }?; + (!attachments.contains(&table)).then(|| { + format!( + "migration {} introduces {table}.community_id without attach_community_write_fence('{table}'::regclass)", + migration.version + ) + }) + }) + }) + .collect() + } + + fn community_write_fence_attachment_violations(sql: &str) -> Vec { + let statements = split_sql_statements(sql); + let attachments = statements + .iter() + .filter_map(|statement| { + let normalized = normalize_sql(statement); + if !normalized.contains("attach_community_write_fence(") { + return None; + } + let call = normalized.split_once("attach_community_write_fence(")?.1; + let table = call + .split(')') + .next()? + .trim() + .split("::") + .next()? + .trim_matches(|ch| ch == '\'' || ch == '"') + .rsplit('.') + .next()? + .to_owned(); + (!table.is_empty()).then_some(table) + }) + .collect::>(); + statements + .into_iter() + .filter_map(|statement| { + let normalized = normalize_sql(&statement); + let table = if normalized.starts_with("create table") + && normalized.contains("community_id") + && !normalized.contains(" partition of ") + { + identifier_after_keyword(&statement, "create table") + } else if normalized.starts_with("alter table") + && normalized.contains("add") + && normalized.contains("community_id") + { + identifier_after_keyword(&statement, "alter table") + } else { + None + }?; + (!attachments.contains(&table)).then_some(table) + }) + .collect() + } + fn scoped_constraint_lints(sql: &str, scoped_tables: &BTreeSet) -> Vec { let mut constraints = table_constraints(sql, scoped_tables); constraints.extend(alter_table_constraints(sql, scoped_tables)); @@ -1014,7 +1114,10 @@ mod tests { assert!(deletion.contains("CREATE TABLE community_deletion_retention_exceptions")); assert!(deletion.contains("CREATE TABLE community_serving_write_leases")); assert!(deletion.contains("CREATE TABLE community_deletion_executor_heartbeats")); + assert!(deletion.contains("CREATE FUNCTION assert_community_write_allowed")); assert!(deletion.contains("CREATE FUNCTION enforce_community_write_fence")); + assert!(deletion.contains("CREATE FUNCTION attach_community_write_fence")); + assert!(deletion.contains("community_write_fence_excluded_table")); assert!(deletion.contains("CREATE FUNCTION enforce_community_tombstone")); assert!(deletion.contains("community tombstones are permanent")); assert!(deletion.contains("_operator_global_tables")); @@ -1023,6 +1126,44 @@ mod tests { >>>>>>> 037fcbdab (feat: add durable community deletion worker) } + #[test] + fn post_deletion_migrations_attach_every_new_community_write_fence() { + let violations = migrations_missing_community_write_fence_attachment(); + assert!( + violations.is_empty(), + "migrations after 0027 must explicitly attach every new community write fence:\n{}", + violations.join("\n") + ); + } + + #[test] + fn community_write_fence_attachment_lint_covers_create_and_alter() { + let missing = r#" + CREATE TABLE created_late ( + community_id UUID NOT NULL, + id UUID NOT NULL + ); + CREATE TABLE altered_late (id UUID NOT NULL); + ALTER TABLE altered_late ADD COLUMN community_id UUID NOT NULL; + "#; + assert_eq!( + community_write_fence_attachment_violations(missing), + vec!["created_late", "altered_late"] + ); + + let attached = r#" + CREATE TABLE created_late ( + community_id UUID NOT NULL, + id UUID NOT NULL + ); + SELECT attach_community_write_fence('created_late'::regclass); + CREATE TABLE altered_late (id UUID NOT NULL); + ALTER TABLE altered_late ADD COLUMN community_id UUID NOT NULL; + SELECT attach_community_write_fence('altered_late'::regclass); + "#; + assert!(community_write_fence_attachment_violations(attached).is_empty()); + } + #[test] fn migration_lint_detects_tables_missing_community_id_by_default() { let sql = r#" @@ -1385,5 +1526,163 @@ mod tests { search_expression.contains("ELSE NULL::tsvector"), "fresh installs must default non-allowlisted kinds to NULL: {search_expression}" ); + + let active_a = uuid::Uuid::new_v4(); + let active_b = uuid::Uuid::new_v4(); + let to_fence = uuid::Uuid::new_v4(); + for (community, label) in [ + (active_a, "active-a"), + (active_b, "active-b"), + (to_fence, "to-fence"), + ] { + sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") + .bind(community) + .bind(format!("late-fence-{label}-{}.example", community.simple())) + .execute(&pool) + .await + .expect("insert late-table test community"); + } + sqlx::query( + "CREATE TABLE late_created_scoped (\ + community_id UUID NOT NULL, id BIGINT PRIMARY KEY, value TEXT NOT NULL\ + )", + ) + .execute(&pool) + .await + .expect("create late scoped table"); + sqlx::query("SELECT attach_community_write_fence('late_created_scoped'::regclass)") + .execute(&pool) + .await + .expect("attach late create fence"); + sqlx::query("CREATE TABLE late_altered_scoped (id BIGINT PRIMARY KEY)") + .execute(&pool) + .await + .expect("create table before late alter"); + sqlx::query("ALTER TABLE late_altered_scoped ADD COLUMN community_id UUID NOT NULL") + .execute(&pool) + .await + .expect("add late community id"); + sqlx::query("SELECT attach_community_write_fence('late_altered_scoped'::regclass)") + .execute(&pool) + .await + .expect("attach late alter fence"); + let attached: Vec = sqlx::query_scalar( + "SELECT c.relname FROM pg_trigger trigger \ + JOIN pg_class c ON c.oid = trigger.tgrelid \ + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid \ + WHERE c.relname IN ('late_created_scoped', 'late_altered_scoped') \ + AND procedure.proname = 'enforce_community_write_fence' \ + AND NOT trigger.tgisinternal ORDER BY c.relname", + ) + .fetch_all(&pool) + .await + .expect("read late trigger catalog"); + assert_eq!(attached, vec!["late_altered_scoped", "late_created_scoped"]); + let malformed_fence_triggers: i64 = sqlx::query_scalar( + "SELECT count(*)::BIGINT FROM pg_trigger trigger \ + JOIN pg_class c ON c.oid = trigger.tgrelid \ + JOIN pg_proc procedure ON procedure.oid = trigger.tgfoid \ + WHERE c.relname IN ('late_created_scoped', 'late_altered_scoped') \ + AND procedure.proname = 'enforce_community_write_fence' \ + AND NOT trigger.tgisinternal \ + AND (trigger.tgenabled <> 'O' OR (trigger.tgtype & 31) <> 31)", + ) + .fetch_one(&pool) + .await + .expect("validate late trigger mode and operations"); + assert_eq!(malformed_fence_triggers, 0); + + sqlx::query( + "INSERT INTO late_created_scoped (community_id, id, value) \ + VALUES ($1, 1, 'same'), ($2, 2, 'source-fenced'), \ + ($1, 3, 'destination-fenced'), ($1, 4, 'opposite-a'), \ + ($3, 5, 'opposite-b')", + ) + .bind(active_a) + .bind(to_fence) + .bind(active_b) + .execute(&pool) + .await + .expect("seed late table while communities active"); + sqlx::query("UPDATE late_created_scoped SET value = 'same-ok' WHERE id = 1") + .execute(&pool) + .await + .expect("same-tenant active update"); + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 1") + .bind(active_b) + .execute(&pool) + .await + .expect("active-to-active update"); + + let mut fence_connection = pool.acquire().await.expect("fence connection"); + sqlx::query("BEGIN") + .execute(&mut *fence_connection) + .await + .expect("begin direct fence"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '1', true)", + ) + .bind(to_fence.to_string()) + .execute(&mut *fence_connection) + .await + .expect("authorize direct fence"); + sqlx::query( + "UPDATE communities SET deletion_state = 'fenced', \ + deletion_fence_generation = 1, archived_at = now() WHERE id = $1", + ) + .bind(to_fence) + .execute(&mut *fence_connection) + .await + .expect("fence test destination"); + sqlx::query("COMMIT") + .execute(&mut *fence_connection) + .await + .expect("commit direct fence"); + + let active_to_fenced = + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 3") + .bind(to_fence) + .execute(&pool) + .await + .expect_err("active to fenced destination must fail"); + assert!(active_to_fenced + .to_string() + .contains("community write fenced")); + let fenced_to_active = + sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 2") + .bind(active_a) + .execute(&pool) + .await + .expect_err("fenced source to active destination must fail"); + assert!(fenced_to_active + .to_string() + .contains("community write fenced")); + let row_locations: Vec<(i64, uuid::Uuid)> = sqlx::query_as( + "SELECT id, community_id FROM late_created_scoped WHERE id IN (2, 3) ORDER BY id", + ) + .fetch_all(&pool) + .await + .expect("failed moves preserve row location"); + assert_eq!(row_locations, vec![(2, to_fence), (3, active_a)]); + + let move_a = sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 4") + .bind(active_b) + .execute(&pool); + let move_b = sqlx::query("UPDATE late_created_scoped SET community_id = $1 WHERE id = 5") + .bind(active_a) + .execute(&pool); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + let (a, b) = tokio::join!(move_a, move_b); + a.expect("opposite active move A"); + b.expect("opposite active move B"); + }) + .await + .expect("opposite cross-tenant updates must not deadlock"); + + sqlx::query("DROP TABLE late_created_scoped, late_altered_scoped") + .execute(&pool) + .await + .expect("drop late-table fixtures"); } } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index a77e00d7df..372eb13686 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -116,15 +116,6 @@ impl ServingWriteGuard { self.lost.clone() } - /// Begin finalization after the side effect itself has succeeded. - /// - /// This stops lease renewal without releasing the durable row. Call - /// [`Self::finish`] immediately after the result's database finalization so - /// deletion cannot overtake the side effect before its outcome is recorded. - pub fn begin_finalize(&self) { - self.cancel.cancel(); - } - /// Release the lease after the side effect completes. pub async fn finish(mut self) -> Result<()> { self.cancel.cancel(); @@ -871,7 +862,19 @@ async fn execute_stage( "approved storage taxonomy drifted before fencing", )); } - services.store.fence(&token).await?; + match services.store.fence(&token).await { + Ok(_) => {} + Err(buzz_db::DbError::ServingWritesNotDrained { + active_count, + operations, + .. + }) => { + return Err(transient(format!( + "serving writes not drained before fence: count={active_count}, operations={operations:?}" + ))); + } + Err(error) => return Err(error.into()), + } } DeletionStage::Fenced => { services diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index 48ea42a371..144fbf846b 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -1783,6 +1783,21 @@ pub(crate) struct PushContext { pub repo_handle: HydratedRepo, } +#[derive(Default)] +struct FinalizePushHooks { + #[cfg(test)] + post_cas_gate: Option>, + #[cfg(test)] + fail_ref_state_insert: bool, +} + +#[cfg(test)] +#[derive(Default)] +struct PostCasGate { + reached: tokio::sync::Notify, + resume: tokio::sync::Notify, +} + /// Finalize a push request: CAS-commit the new state into the object /// store, derive kind:30618 from the committed manifest, and only then /// build the success response. @@ -1793,6 +1808,17 @@ pub(crate) struct PushContext { /// constructor of a push 2xx, so the seam is structural (not by /// convention). async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { + finalize_push_inner(state, ctx, &FinalizePushHooks::default()).await +} + +async fn finalize_push_inner( + state: &Arc, + ctx: PushContext, + hooks: &FinalizePushHooks, +) -> Response { + #[cfg(not(test))] + let _ = hooks; + // The push fence, part 0 — **a rejected push publishes nothing.** // // `ctx.pack.ok` is false when git aborted the ref updates: either the @@ -1944,8 +1970,10 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { } }; - if let Err(error) = serving_write.finish().await { - warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "failed to release community serving lease after push"); + #[cfg(test)] + if let Some(gate) = &hooks.post_cas_gate { + gate.reached.notify_one(); + gate.resume.notified().await; } // Derived after CAS: kind:30618 ref-state event over the *committed* @@ -1971,7 +1999,7 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { (Some(before), Some(after)) => before != after, _ => true, // first push (parent None) or impossible-shape after key → publish }; - if manifest_changed { + let publication_result: Result<(), String> = if manifest_changed { let inputs = RefStateInputs { repo_id: &ctx.repo_id, head: &success.manifest.head, @@ -1982,11 +2010,23 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { Ok(event) => { // Relay-signed kind:30618 belongs to the same server-resolved // tenant as the git request that committed the pointer. - match state + #[cfg(test)] + let insert_result = if hooks.fail_ref_state_insert { + Err(buzz_db::DbError::InvalidData( + "injected kind:30618 insert failure".to_string(), + )) + } else { + state + .db + .insert_event_with_serving_write_guard(ctx.tenant.community(), &event, None) + .await + }; + #[cfg(not(test))] + let insert_result = state .db - .insert_event(ctx.tenant.community(), &event, None) - .await - { + .insert_event_with_serving_write_guard(ctx.tenant.community(), &event, None) + .await; + match insert_result { Ok((stored, true)) => { // Routed through the guarded send path for uniformity; // the access gate no-ops for this globally-scoped @@ -2003,6 +2043,7 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { manifest = %success.manifest_key, "kind:30618 published (derived after CAS)" ); + Ok(()) } Ok((_, false)) => { info!( @@ -2010,26 +2051,41 @@ async fn finalize_push(state: &Arc, ctx: PushContext) -> Response { repo = %ctx.repo_id, "kind:30618 deduplicated by relay db" ); + Ok(()) } - Err(e) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo_id, - error = %e, - "kind:30618 insert failed; push remains durable in object store" - ); - } + Err(error) => Err(format!("kind:30618 insert failed: {error}")), } } - Err(e) => { - warn!( - owner = %ctx.owner, - repo = %ctx.repo_id, - error = %e, - "kind:30618 build failed; push remains durable in object store" - ); - } + Err(error) => Err(format!("kind:30618 build failed: {error}")), } + } else { + Ok(()) + }; + + // The admitted serving write spans the complete publication attempt. Fence + // acquisition cannot overtake the pointer CAS, durable 30618 insert, or + // local fan-out attempt; only now may the lease be released. + if let Err(error) = serving_write.finish().await { + warn!(owner = %ctx.owner, repo = %ctx.repo, %error, "failed to release community serving lease after push publication"); + return ( + StatusCode::SERVICE_UNAVAILABLE, + "community write lease lost during publication", + ) + .into_response(); + } + if let Err(error) = publication_result { + error!( + owner = %ctx.owner, + repo = %ctx.repo_id, + manifest = %success.manifest_key, + %error, + "push pointer committed but kind:30618 publication failed" + ); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + "push committed but ref-state publication failed; retry", + ) + .into_response(); } // Only now — after CAS commit and (optional) 30618 emission — build @@ -2058,12 +2114,14 @@ pub fn git_router(state: Arc) -> Router { #[cfg(test)] mod track_c_tests { use super::*; + use crate::api::git::hydrate::{hydrate_for_write, HydrationOptions}; use crate::api::git::manifest::Manifest; use buzz_core::CommunityId; use nostr::{EventBuilder, Keys, Kind, Tag}; use std::collections::BTreeMap; use std::io::Write; use std::process::Output; + use tempfile::TempDir; fn oid_sha1() -> String { "cb09a769da1c01f458fa6959d4e8eded38fac8d3".to_string() @@ -2204,6 +2262,292 @@ mod track_c_tests { assert!(remote.join("refs/heads/master").exists()); } + async fn run_finalize_git(repo: &Path, args: &[&str]) -> std::process::Output { + let mut command = Command::new("git"); + command.current_dir(repo).args(args); + harden_git_env(&mut command); + let output = command.output().await.expect("spawn git"); + assert!( + output.status.success(), + "git {args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + output + } + + async fn finalize_test_state() -> (Arc, sqlx::PgPool) { + const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 + let mut config = crate::config::Config::from_env().expect("default config loads"); + config.require_relay_membership = false; + config.redis_url = "redis://127.0.0.1:1".to_string(); + config.database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| TEST_DB_URL.to_string()); + let pool = sqlx::PgPool::connect(&config.database_url) + .await + .expect("connect test DB"); + let db = buzz_db::Db::from_pool(pool.clone()); + db.migrate().await.expect("migrate test DB"); + let redis_pool = deadpool_redis::Config::from_url(&config.redis_url) + .create_pool(Some(deadpool_redis::Runtime::Tokio1)) + .expect("redis pool"); + let pubsub = Arc::new( + buzz_pubsub::PubSubManager::new(&config.redis_url, redis_pool.clone()) + .await + .expect("pubsub manager"), + ); + let audit = buzz_audit::AuditService::new(pool.clone()); + let auth = buzz_auth::AuthService::new(config.auth.clone()); + let search = buzz_search::SearchService::new(pool.clone()); + let workflow_engine = Arc::new(buzz_workflow::WorkflowEngine::new( + db.clone(), + buzz_workflow::WorkflowConfig::default(), + )); + let media_storage = buzz_media::MediaStorage::new(&config.media).expect("media storage"); + let (state, _audit_shutdown) = AppState::new( + config, + db, + redis_pool, + audit, + pubsub, + auth, + search, + workflow_engine, + Keys::generate(), + media_storage, + ); + (Arc::new(state), pool) + } + + async fn approved_deletion( + state: &AppState, + host: &str, + ) -> ( + buzz_db::deletion::DeletionRequest, + buzz_db::deletion::ClaimedDeletion, + ) { + use buzz_db::deletion::{FrozenInventory, StorageManifest, DEFAULT_LEASE_DURATION}; + + let store = state.db.deletion_store(); + let request = store + .submit(host, "git-finalize-test", Some("post-CAS lease regression")) + .await + .expect("submit deletion"); + let inventory = FrozenInventory { + schema: store + .inventory_schema(request.community_id) + .await + .expect("schema inventory"), + storage: StorageManifest { + version: 1, + tenant_keys: Vec::new(), + git_pointer_keys: Vec::new(), + media_sidecar_keys: Vec::new(), + media_upload_keys: Vec::new(), + retained_shared_cas_keys: Vec::new(), + unknown_keys: Vec::new(), + unsupported_version_keys: Vec::new(), + }, + }; + store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze inventory"); + store + .approve(request.id, "git-finalize-test", None) + .await + .expect("approve deletion"); + let claim = store + .claim_specific(request.id, "git-finalize-test", DEFAULT_LEASE_DURATION) + .await + .expect("claim deletion") + .expect("won deletion claim"); + (request, claim) + } + + async fn pushed_context( + state: &AppState, + community: CommunityId, + host: &str, + owner: String, + repo: String, + pusher: nostr::PublicKey, + scratch: &Path, + ) -> PushContext { + let tenant = TenantContext::resolved(community, host); + let (hydrated, parent_state) = hydrate_for_write( + &state.git_store, + &tenant, + &owner, + &repo, + HydrationOptions { + pack_cache: &state.git_pack_cache, + scratch_dir: scratch, + max_pack_bytes: 1024 * 1024, + max_repo_bytes: 2 * 1024 * 1024, + }, + ) + .await + .expect("hydrate empty test repo"); + let source = scratch.join("source"); + tokio::fs::create_dir(&source) + .await + .expect("source directory"); + run_finalize_git(&source, &["init", "--quiet", "--initial-branch=main"]).await; + run_finalize_git(&source, &["config", "user.email", "finalize@test"]).await; + run_finalize_git(&source, &["config", "user.name", "finalize"]).await; + tokio::fs::write(source.join("file.txt"), b"committed\n") + .await + .expect("write source file"); + run_finalize_git(&source, &["add", "file.txt"]).await; + run_finalize_git(&source, &["commit", "--quiet", "-m", "committed"]).await; + let remote = hydrated.path().to_str().expect("hydrated path utf8"); + run_finalize_git(&source, &["push", "--quiet", remote, "main"]).await; + + PushContext { + pack: PackOutput { + stdout: b"push-ok".to_vec(), + ok: true, + }, + parent_state, + owner, + repo: repo.clone(), + repo_id: repo, + pusher, + tenant, + repo_handle: hydrated, + } + } + + #[tokio::test] + #[ignore = "requires Postgres and MinIO"] + async fn finalize_push_holds_serving_lease_through_post_cas_publication() { + let (state, pool) = finalize_test_state().await; + let host = format!("git-finalize-{}.example", uuid::Uuid::new_v4().simple()); + let community = state + .db + .ensure_configured_community(&host) + .await + .expect("create test community") + .id; + let (request, claim) = approved_deletion(&state, &host).await; + let scratch = TempDir::new().expect("scratch"); + let owner = format!("owner-{}", uuid::Uuid::new_v4().simple()); + let repo = format!("repo-{}", uuid::Uuid::new_v4().simple()); + let ctx = pushed_context( + &state, + community, + &host, + owner, + repo.clone(), + Keys::generate().public_key(), + scratch.path(), + ) + .await; + let gate = Arc::new(PostCasGate::default()); + let hooks = FinalizePushHooks { + post_cas_gate: Some(Arc::clone(&gate)), + fail_ref_state_insert: false, + }; + let finalize_state = Arc::clone(&state); + let finalize = + tokio::spawn(async move { finalize_push_inner(&finalize_state, ctx, &hooks).await }); + + gate.reached.notified().await; + let error = state + .db + .deletion_store() + .fence(&claim.lease) + .await + .expect_err("post-CAS serving lease must block fence"); + assert!(matches!( + error, + buzz_db::DbError::ServingWritesNotDrained { .. } + )); + assert!(state + .db + .deletion_store() + .is_serving_active(community) + .await + .expect("community remains active")); + + gate.resume.notify_one(); + let response = finalize.await.expect("finalize task"); + assert_eq!(response.status(), StatusCode::OK); + let mut query = buzz_db::event::EventQuery::for_community(community); + query.kinds = Some(vec![30_618]); + query.d_tag = Some(repo); + let events = state.db.query_events(&query).await.expect("query 30618"); + assert_eq!(events.len(), 1, "kind:30618 must be durable before release"); + assert!(state + .db + .deletion_store() + .serving_writes_drained(community) + .await + .expect("serving lease released")); + let generation = state + .db + .deletion_store() + .fence(&claim.lease) + .await + .expect("fence after publication"); + assert_eq!(generation, 1); + assert_eq!( + state + .db + .deletion_store() + .get(request.id) + .await + .expect("fenced request") + .stage, + buzz_db::deletion::DeletionStage::Fenced + ); + drop(state); + pool.close().await; + } + + #[tokio::test] + #[ignore = "requires Postgres and MinIO"] + async fn finalize_push_db_failure_after_cas_is_not_success_and_releases_lease() { + let (state, pool) = finalize_test_state().await; + let host = format!( + "git-finalize-fail-{}.example", + uuid::Uuid::new_v4().simple() + ); + let community = state + .db + .ensure_configured_community(&host) + .await + .expect("create test community") + .id; + let scratch = TempDir::new().expect("scratch"); + let ctx = pushed_context( + &state, + community, + &host, + format!("owner-{}", uuid::Uuid::new_v4().simple()), + format!("repo-{}", uuid::Uuid::new_v4().simple()), + Keys::generate().public_key(), + scratch.path(), + ) + .await; + let hooks = FinalizePushHooks { + post_cas_gate: None, + fail_ref_state_insert: true, + }; + + let response = finalize_push_inner(&state, ctx, &hooks).await; + assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(state + .db + .deletion_store() + .serving_writes_drained(community) + .await + .expect("serving lease released on failure")); + drop(state); + pool.close().await; + } + /// A gzip-encoded request body is transparently inflated before it /// reaches the git subprocess. Git's smart-HTTP client gzips the /// upload-pack/receive-pack request body past a size threshold (fires diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index 80fe5a678a..cc6fe74718 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -201,6 +201,12 @@ async fn main() -> anyhow::Result<()> { error!("Failed to ensure partitions: {e}"); } + db.validate_deletion_catalog().await.map_err(|e| { + error!("Community deletion catalog validation failed: {e}"); + anyhow::anyhow!("Community deletion catalog is unsafe: {e}") + })?; + info!("Community deletion catalog and write fences verified"); + // Freshness fence probe: cursor pages route to the replica only for // history the probe has verified as fully replayed. Deliberately AFTER // the migration decision: spawn_fence_probe first verifies the diff --git a/crates/buzz-relay/src/push_runtime.rs b/crates/buzz-relay/src/push_runtime.rs index d6b7f04b91..4946b248c6 100644 --- a/crates/buzz-relay/src/push_runtime.rs +++ b/crates/buzz-relay/src/push_runtime.rs @@ -460,7 +460,6 @@ async fn deliver_one( return; } }; - serving_write.begin_finalize(); match response { Ok(r) if r.status().is_success() => match r.json::().await { Ok(DeliveryResponse::Accepted) => { diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index 400ed1dfe3..f0daa5a73d 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -376,22 +376,30 @@ async fn readiness_handler(State(state): State>) -> impl IntoRespo } let check = async { - let (pg_ok, redis_ok) = tokio::join!(state.db.ping(), async { - state.redis_pool.get().await.is_ok() - },); - (pg_ok, redis_ok) + let (pg_ok, redis_ok, deletion_catalog_ok) = tokio::join!( + state.db.ping(), + async { state.redis_pool.get().await.is_ok() }, + async { state.db.validate_deletion_catalog().await.is_ok() }, + ); + (pg_ok, redis_ok, deletion_catalog_ok) }; - let (pg_ok, redis_ok) = tokio::time::timeout(Duration::from_secs(2), check) - .await - .unwrap_or((false, false)); + let (pg_ok, redis_ok, deletion_catalog_ok) = + tokio::time::timeout(Duration::from_secs(2), check) + .await + .unwrap_or((false, false, false)); - if pg_ok && redis_ok { + if pg_ok && redis_ok && deletion_catalog_ok { (StatusCode::OK, Json(json!({"status": "ready"}))).into_response() } else { ( StatusCode::SERVICE_UNAVAILABLE, - Json(json!({"status": "not_ready", "postgres": pg_ok, "redis": redis_ok})), + Json(json!({ + "status": "not_ready", + "postgres": pg_ok, + "redis": redis_ok, + "deletion_catalog": deletion_catalog_ok + })), ) .into_response() } diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 1f15a60f1e..b31690f533 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -133,24 +133,32 @@ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ SELECT hashtextextended('buzz-community-deletion:' || target::text, 0) $$; -CREATE FUNCTION enforce_community_write_fence() RETURNS TRIGGER +-- Keep the deletion control plane writable while its target tenant is fenced. +-- This predicate is the single SQL source of truth used by attachment and live +-- catalog validation. +CREATE FUNCTION community_write_fence_excluded_table(target NAME) RETURNS BOOLEAN +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT target::TEXT = ANY (ARRAY[ + 'community_deletion_requests', + 'community_deletion_approvals', + 'community_deletion_checkpoints', + 'community_deletion_retention_exceptions', + 'community_serving_write_leases', + 'community_deletion_executor_heartbeats' + ]::TEXT[]) +$$; + +CREATE FUNCTION assert_community_write_allowed(target UUID) RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE - target UUID; lifecycle TEXT; generation BIGINT; executor_community TEXT; executor_generation TEXT; BEGIN - IF TG_OP = 'INSERT' THEN - target := NEW.community_id; - ELSE - target := OLD.community_id; - END IF; - -- Nullable operator-attribution rows without a tenant are unrelated. IF target IS NULL THEN - RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; + RETURN; END IF; PERFORM pg_advisory_xact_lock_shared(community_deletion_lock_key(target)); @@ -163,18 +171,43 @@ BEGIN USING ERRCODE = 'object_not_in_prerequisite_state'; END IF; + -- Authorization is evaluated independently for every community checked. executor_community := current_setting('buzz.deletion_executor_community', true); executor_generation := current_setting('buzz.deletion_fence_generation', true); - IF executor_community = target::text + IF executor_community = target::TEXT AND executor_generation ~ '^[0-9]+$' AND executor_generation::BIGINT = generation THEN - RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; + RETURN; END IF; IF lifecycle <> 'active' THEN RAISE EXCEPTION 'community write fenced: community % generation %', target, generation USING ERRCODE = 'object_not_in_prerequisite_state'; END IF; +END +$$; + +CREATE FUNCTION enforce_community_write_fence() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + PERFORM assert_community_write_allowed(NEW.community_id); + ELSIF TG_OP = 'DELETE' THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id IS NOT DISTINCT FROM NEW.community_id THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id IS NULL THEN + PERFORM assert_community_write_allowed(NEW.community_id); + ELSIF NEW.community_id IS NULL THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id < NEW.community_id THEN + PERFORM assert_community_write_allowed(OLD.community_id); + PERFORM assert_community_write_allowed(NEW.community_id); + ELSE + PERFORM assert_community_write_allowed(NEW.community_id); + PERFORM assert_community_write_allowed(OLD.community_id); + END IF; + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; END $$; @@ -224,16 +257,60 @@ CREATE TRIGGER communities_deletion_tombstone BEFORE UPDATE OR DELETE ON communities FOR EACH ROW EXECUTE FUNCTION enforce_community_tombstone(); +-- Attach the universal fence to one community-scoped relation. Future +-- migrations must invoke this helper explicitly after CREATE/ALTER introduces +-- community_id; the migration lint enforces that contract. +CREATE FUNCTION attach_community_write_fence(target REGCLASS) RETURNS VOID +LANGUAGE plpgsql AS $$ +DECLARE + relation_name NAME; +BEGIN + SELECT c.relname + INTO relation_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = target + AND n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition; + IF NOT FOUND THEN + RAISE EXCEPTION 'community write fence target % is not a public table', target + USING ERRCODE = 'wrong_object_type'; + END IF; + IF community_write_fence_excluded_table(relation_name) THEN + RETURN; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = target AND attname = 'community_id' AND NOT attisdropped + ) THEN + RAISE EXCEPTION 'community write fence target % has no community_id', target + USING ERRCODE = 'undefined_column'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgrelid = target + AND tgname = 'community_write_fence_' || relation_name + AND NOT tgisinternal + ) THEN + EXECUTE format( + 'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %s ' + 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', + 'community_write_fence_' || relation_name, + target + ); + END IF; +END +$$; + -- Attach the universal fence to every existing table carrying community_id, -- including deployment-private sidecars whose community_id is provenance. --- Deletion control-plane tables are excluded: they must remain writable while --- the target tenant is fenced. DO $$ DECLARE - table_name TEXT; + target REGCLASS; BEGIN - FOR table_name IN - SELECT c.relname + FOR target IN + SELECT c.oid::REGCLASS FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace JOIN pg_attribute a ON a.attrelid = c.oid @@ -242,21 +319,10 @@ BEGIN AND NOT c.relispartition AND a.attname = 'community_id' AND NOT a.attisdropped - AND c.relname NOT IN ( - 'community_deletion_requests', - 'community_deletion_approvals', - 'community_deletion_checkpoints', - 'community_deletion_retention_exceptions', - 'community_serving_write_leases', - 'community_deletion_executor_heartbeats' - ) + AND NOT community_write_fence_excluded_table(c.relname) + ORDER BY c.oid::REGCLASS::TEXT LOOP - EXECUTE format( - 'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %I ' - 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', - 'community_write_fence_' || table_name, - table_name - ); + PERFORM attach_community_write_fence(target); END LOOP; END $$; diff --git a/schema/schema.sql b/schema/schema.sql index 3d9f53766e..6e9901c904 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1198,37 +1198,85 @@ CREATE FUNCTION community_deletion_lock_key(target UUID) RETURNS BIGINT LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ SELECT hashtextextended('buzz-community-deletion:' || target::text, 0) $$; -CREATE FUNCTION enforce_community_write_fence() RETURNS TRIGGER +-- Keep the deletion control plane writable while its target tenant is fenced. +-- This predicate is the single SQL source of truth used by attachment and live +-- catalog validation. +CREATE FUNCTION community_write_fence_excluded_table(target NAME) RETURNS BOOLEAN +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ + SELECT target::TEXT = ANY (ARRAY[ + 'community_deletion_requests', + 'community_deletion_approvals', + 'community_deletion_checkpoints', + 'community_deletion_retention_exceptions', + 'community_serving_write_leases', + 'community_deletion_executor_heartbeats' + ]::TEXT[]) +$$; + +CREATE FUNCTION assert_community_write_allowed(target UUID) RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE - target UUID; lifecycle TEXT; generation BIGINT; executor_community TEXT; executor_generation TEXT; BEGIN - IF TG_OP = 'INSERT' THEN target := NEW.community_id; ELSE target := OLD.community_id; END IF; - IF target IS NULL THEN RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; END IF; + -- Nullable operator-attribution rows without a tenant are unrelated. + IF target IS NULL THEN + RETURN; + END IF; + PERFORM pg_advisory_xact_lock_shared(community_deletion_lock_key(target)); - SELECT deletion_state, deletion_fence_generation INTO lifecycle, generation - FROM communities WHERE id = target; + SELECT deletion_state, deletion_fence_generation + INTO lifecycle, generation + FROM communities + WHERE id = target; IF NOT FOUND THEN RAISE EXCEPTION 'community write rejected: community % is missing', target USING ERRCODE = 'object_not_in_prerequisite_state'; END IF; + + -- Authorization is evaluated independently for every community checked. executor_community := current_setting('buzz.deletion_executor_community', true); executor_generation := current_setting('buzz.deletion_fence_generation', true); - IF executor_community = target::text AND executor_generation ~ '^[0-9]+$' + IF executor_community = target::TEXT + AND executor_generation ~ '^[0-9]+$' AND executor_generation::BIGINT = generation THEN - RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; + RETURN; END IF; + IF lifecycle <> 'active' THEN RAISE EXCEPTION 'community write fenced: community % generation %', target, generation USING ERRCODE = 'object_not_in_prerequisite_state'; END IF; +END +$$; + +CREATE FUNCTION enforce_community_write_fence() RETURNS TRIGGER +LANGUAGE plpgsql AS $$ +BEGIN + IF TG_OP = 'INSERT' THEN + PERFORM assert_community_write_allowed(NEW.community_id); + ELSIF TG_OP = 'DELETE' THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id IS NOT DISTINCT FROM NEW.community_id THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id IS NULL THEN + PERFORM assert_community_write_allowed(NEW.community_id); + ELSIF NEW.community_id IS NULL THEN + PERFORM assert_community_write_allowed(OLD.community_id); + ELSIF OLD.community_id < NEW.community_id THEN + PERFORM assert_community_write_allowed(OLD.community_id); + PERFORM assert_community_write_allowed(NEW.community_id); + ELSE + PERFORM assert_community_write_allowed(NEW.community_id); + PERFORM assert_community_write_allowed(OLD.community_id); + END IF; + RETURN CASE WHEN TG_OP = 'DELETE' THEN OLD ELSE NEW END; END $$; + CREATE FUNCTION enforce_community_tombstone() RETURNS TRIGGER LANGUAGE plpgsql AS $$ DECLARE @@ -1259,22 +1307,72 @@ END $$; CREATE TRIGGER communities_deletion_tombstone BEFORE UPDATE OR DELETE ON communities FOR EACH ROW EXECUTE FUNCTION enforce_community_tombstone(); +-- Attach the universal fence to one community-scoped relation. Future +-- migrations must invoke this helper explicitly after CREATE/ALTER introduces +-- community_id; the migration lint enforces that contract. +CREATE FUNCTION attach_community_write_fence(target REGCLASS) RETURNS VOID +LANGUAGE plpgsql AS $$ +DECLARE + relation_name NAME; +BEGIN + SELECT c.relname + INTO relation_name + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.oid = target + AND n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition; + IF NOT FOUND THEN + RAISE EXCEPTION 'community write fence target % is not a public table', target + USING ERRCODE = 'wrong_object_type'; + END IF; + IF community_write_fence_excluded_table(relation_name) THEN + RETURN; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_attribute + WHERE attrelid = target AND attname = 'community_id' AND NOT attisdropped + ) THEN + RAISE EXCEPTION 'community write fence target % has no community_id', target + USING ERRCODE = 'undefined_column'; + END IF; + IF NOT EXISTS ( + SELECT 1 FROM pg_trigger + WHERE tgrelid = target + AND tgname = 'community_write_fence_' || relation_name + AND NOT tgisinternal + ) THEN + EXECUTE format( + 'CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %s ' + 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', + 'community_write_fence_' || relation_name, + target + ); + END IF; +END +$$; + +-- Attach the universal fence to every existing table carrying community_id, +-- including deployment-private sidecars whose community_id is provenance. DO $$ -DECLARE table_name TEXT; +DECLARE + target REGCLASS; BEGIN - FOR table_name IN - SELECT c.relname FROM pg_class c - JOIN pg_namespace n ON n.oid = c.relnamespace - JOIN pg_attribute a ON a.attrelid = c.oid - WHERE n.nspname = 'public' AND c.relkind IN ('r', 'p') AND NOT c.relispartition - AND a.attname = 'community_id' AND NOT a.attisdropped - AND c.relname NOT IN ('community_deletion_requests', 'community_deletion_approvals', - 'community_deletion_checkpoints', 'community_deletion_retention_exceptions', - 'community_serving_write_leases', 'community_deletion_executor_heartbeats') + FOR target IN + SELECT c.oid::REGCLASS + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_attribute a ON a.attrelid = c.oid + WHERE n.nspname = 'public' + AND c.relkind IN ('r', 'p') + AND NOT c.relispartition + AND a.attname = 'community_id' + AND NOT a.attisdropped + AND NOT community_write_fence_excluded_table(c.relname) + ORDER BY c.oid::REGCLASS::TEXT LOOP - EXECUTE format('CREATE TRIGGER %I BEFORE INSERT OR UPDATE OR DELETE ON %I ' - 'FOR EACH ROW EXECUTE FUNCTION enforce_community_write_fence()', - 'community_write_fence_' || table_name, table_name); + PERFORM attach_community_write_fence(target); END LOOP; END $$; From 7fa4b7574387f4909a1e72aa46788e747863fb82 Mon Sep 17 00:00:00 2001 From: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Date: Fri, 31 Jul 2026 21:09:52 -0700 Subject: [PATCH 07/51] fix(e2e): make live relay checks topology-correct Give unarchive membership signals unique event ids, derive live-test authorities and media URLs from the configured relay, use metadata-free PNG bytes, respect community-scoped Git pointer keys, and let the subscription-cap proof coexist with the independent admission quota. Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> --- .../buzz-relay/src/handlers/side_effects.rs | 49 +++++--- crates/buzz-test-client/tests/e2e_git.rs | 63 ++++++++-- .../tests/e2e_media_extended.rs | 25 ++-- crates/buzz-test-client/tests/e2e_relay.rs | 114 ++++++++++-------- deploy/local/build-and-deploy.sh | 24 ++-- 5 files changed, 186 insertions(+), 89 deletions(-) diff --git a/crates/buzz-relay/src/handlers/side_effects.rs b/crates/buzz-relay/src/handlers/side_effects.rs index 98f8a9aa84..68c73ab277 100644 --- a/crates/buzz-relay/src/handlers/side_effects.rs +++ b/crates/buzz-relay/src/handlers/side_effects.rs @@ -904,6 +904,27 @@ pub async fn emit_membership_notification( target_pubkey: &[u8], actor_pubkey: &[u8], notification_kind: u32, +) -> anyhow::Result<()> { + emit_membership_notification_with_nonce( + tenant, + state, + channel_id, + target_pubkey, + actor_pubkey, + notification_kind, + None, + ) + .await +} + +async fn emit_membership_notification_with_nonce( + tenant: &TenantContext, + state: &Arc, + channel_id: Uuid, + target_pubkey: &[u8], + actor_pubkey: &[u8], + notification_kind: u32, + nonce: Option<&str>, ) -> anyhow::Result<()> { let target_hex = hex::encode(target_pubkey); let actor_hex = hex::encode(actor_pubkey); @@ -931,8 +952,15 @@ pub async fn emit_membership_notification( }) .to_string(); + let mut tags = vec![p_tag, h_tag]; + if let Some(nonce) = nonce { + tags.push( + Tag::parse(["nonce", nonce]) + .map_err(|e| anyhow::anyhow!("failed to build nonce tag: {e}"))?, + ); + } let event = EventBuilder::new(Kind::Custom(notification_kind as u16), content) - .tags([p_tag, h_tag]) + .tags(tags) .sign_with_keys(&state.relay_keypair) .map_err(|e| anyhow::anyhow!("failed to sign membership notification: {e}"))?; @@ -1605,28 +1633,21 @@ async fn handle_edit_metadata( // Resubscribe connected agents after restore: archiving evicts their // live subscriptions (CLOSED "channel access revoked") and unarchive // otherwise emits no signal that makes a connected agent resubscribe. - // We reuse the member_added notification (44100) purely as a resubscribe - // trigger — no membership actually changed here — because it flows on the - // agent's always-live global membership subscription, the same path - // remove/re-add uses to recover. Humans self-heal via the re-emitted - // kind:39000 discovery, so this is intentionally agent-scoped. - // - // Known limitation: emit_membership_notification builds a created_at=now - // event with no nonce, and insert_event skips fan-out on a duplicate id. - // Four sub-second toggles (archive->unarchive->archive->unarchive) on the - // same channel by the same actor could collide ids and skip a fan-out. - // Not reachable in practice — unarchive has a single human-driven caller; - // the reaper only auto-archives — so we don't engineer around it. + // We reuse member_added (44100) as the existing global resubscribe + // trigger. Each unarchive carries the accepted edit event id as a + // nonce so it cannot collide with the original member-added event (or + // another unarchive) within Nostr's one-second timestamp precision. for member in state.db.get_members(tenant.community(), channel_id).await? { - if let Err(e) = emit_membership_notification( + if let Err(e) = emit_membership_notification_with_nonce( tenant, state, channel_id, &member.pubkey, &actor_bytes, KIND_MEMBER_ADDED_NOTIFICATION, + Some(&event.id.to_hex()), ) .await { diff --git a/crates/buzz-test-client/tests/e2e_git.rs b/crates/buzz-test-client/tests/e2e_git.rs index 3c82e31764..6b73fe2e77 100644 --- a/crates/buzz-test-client/tests/e2e_git.rs +++ b/crates/buzz-test-client/tests/e2e_git.rs @@ -189,16 +189,62 @@ impl GitS3Probe { Self { bucket } } - fn pointer_key(owner: &str, repo: &str) -> String { + fn configured_pointer_key(owner: &str, repo: &str) -> Option { let repo = repo.strip_suffix(".git").unwrap_or(repo); - if let Ok(community) = std::env::var("BUZZ_E2E_GIT_COMMUNITY_ID") { - return format!("repos/{community}/{owner}/{repo}/pointer"); + std::env::var("BUZZ_E2E_GIT_COMMUNITY_ID") + .ok() + .map(|community| format!("repos/{community}/{owner}/{repo}/pointer")) + } + + async fn find_pointer_key(&self, owner: &str, repo: &str) -> Option { + if let Some(key) = Self::configured_pointer_key(owner, repo) { + return Some(key); + } + + // Git pointers are community-scoped. Discover the exact pointer created + // for this random owner/repo rather than probing the obsolete unscoped + // `repos/{owner}/{repo}/pointer` layout. The optional env override above + // remains useful when a harness already knows the tenant UUID. + let repo = repo.strip_suffix(".git").unwrap_or(repo); + let suffix = format!("/{owner}/{repo}/pointer"); + let mut continuation = None; + let mut matches = Vec::new(); + loop { + let (page, _) = self + .bucket + .list_page( + "repos/".to_string(), + None, + continuation.take(), + None, + Some(1000), + ) + .await + .expect("list Git pointer namespace"); + matches.extend( + page.contents + .into_iter() + .map(|object| object.key) + .filter(|key| key.ends_with(&suffix)), + ); + if !page.is_truncated { + break; + } + continuation = page.next_continuation_token; + assert!( + continuation.is_some(), + "truncated Git pointer listing must include a continuation token" + ); } - format!("repos/{owner}/{repo}/pointer") + assert!( + matches.len() <= 1, + "random E2E owner/repo resolved multiple community pointers: {matches:?}" + ); + matches.pop() } async fn pointer(&self, owner: &str, repo: &str) -> Option { - let key = Self::pointer_key(owner, repo); + let key = self.find_pointer_key(owner, repo).await?; match self.bucket.get_object(&key).await { Ok(resp) => { let etag = resp @@ -225,10 +271,9 @@ impl GitS3Probe { } tokio::time::sleep(Duration::from_millis(250)).await; } - panic!( - "S3 manifest pointer {} never appeared; git may have fallen back to disk", - Self::pointer_key(owner, repo) - ); + let expected = Self::configured_pointer_key(owner, repo) + .unwrap_or_else(|| format!("repos//{owner}/{repo}/pointer")); + panic!("S3 manifest pointer {expected} never appeared") } async fn assert_manifest_exists(&self, digest: &str) { diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 955bd9d6c4..156bfc7ffe 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -19,6 +19,11 @@ fn relay_ws_url() -> String { .replace("https://", "wss://") } +fn relay_authority() -> String { + let url = url::Url::parse(&relay_http_url()).expect("relay HTTP URL"); + url[url::Position::BeforeHost..url::Position::AfterPort].to_string() +} + fn http_client() -> Client { Client::builder() .timeout(Duration::from_secs(15)) @@ -98,15 +103,13 @@ fn tiny_jpeg() -> Vec { } fn tiny_png() -> Vec { - // Valid 2x2 red PNG generated by ffmpeg + // Valid, metadata-free 2x2 red PNG. vec![ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x08, 0x02, 0x00, 0x00, 0x00, 0xfd, - 0xd4, 0x9a, 0x73, 0x00, 0x00, 0x00, 0x09, 0x70, 0x48, 0x59, 0x73, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x4f, 0x25, 0xc4, 0xd6, 0x00, 0x00, 0x00, 0x10, 0x49, 0x44, - 0x41, 0x54, 0x78, 0x9c, 0x63, 0xfc, 0xc3, 0x00, 0x02, 0x2c, 0x60, 0x92, 0x01, 0x00, 0x0d, - 0x04, 0x01, 0x02, 0xbf, 0x50, 0x15, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, - 0xae, 0x42, 0x60, 0x82, + 0xd4, 0x9a, 0x73, 0x00, 0x00, 0x00, 0x10, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0xf8, + 0xcf, 0xc0, 0x00, 0x44, 0x0c, 0x10, 0x0a, 0x00, 0x1f, 0xee, 0x03, 0xfd, 0x8b, 0x5f, 0x14, + 0xd4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, ] } @@ -371,7 +374,7 @@ async fn test_auth_server_tag_correct() { Tag::parse(["t", "upload"]).unwrap(), Tag::parse(["x", &sha256]).unwrap(), Tag::parse(["expiration", &(now + 300).to_string()]).unwrap(), - Tag::parse(["server", "localhost:3000"]).unwrap(), + Tag::parse(["server", &relay_authority()]).unwrap(), ], ); let resp = upload_with_auth(&client, &auth, &sha256, &jpeg).await; @@ -558,6 +561,10 @@ async fn test_ws_valid_imeta() { let resp = upload(&http, &keys, &jpeg).await; assert_eq!(resp.status(), 200); + let upload_desc: serde_json::Value = resp.json().await.expect("upload descriptor"); + let media_url = upload_desc["url"].as_str().expect("uploaded media URL"); + let media_size = upload_desc["size"].as_u64().expect("uploaded media size"); + // Connect via WebSocket let mut client = BuzzTestClient::connect(&relay_ws_url(), &keys) .await @@ -569,10 +576,10 @@ async fn test_ws_valid_imeta() { Tag::parse(["h", &channel_id]).unwrap(), Tag::parse([ "imeta", - &format!("url http://localhost:3000/media/{sha256}.jpg"), + &format!("url {media_url}"), "m image/jpeg", &format!("x {sha256}"), - "size 347", + &format!("size {media_size}"), ]) .unwrap(), ]) diff --git a/crates/buzz-test-client/tests/e2e_relay.rs b/crates/buzz-test-client/tests/e2e_relay.rs index 5b9a50b5b1..a8adcc15ef 100644 --- a/crates/buzz-test-client/tests/e2e_relay.rs +++ b/crates/buzz-test-client/tests/e2e_relay.rs @@ -120,7 +120,12 @@ async fn seed_relay_member(host: &str, keys: &Keys, role: &str) { } async fn seed_relay_owner(keys: &Keys) { - seed_relay_member("localhost:3000", keys, "owner").await; + seed_relay_member(&relay_authority(), keys, "owner").await; +} + +fn relay_authority() -> String { + let url = url::Url::parse(&relay_http_url()).expect("relay HTTP URL"); + url[url::Position::BeforeHost..url::Position::AfterPort].to_string() } fn http_origin_for_host(host: &str) -> String { @@ -315,7 +320,7 @@ async fn test_invite_claim_rejects_invalid_code() { #[ignore] async fn test_invite_mint_requires_owner_or_admin() { let member = Keys::generate(); - seed_relay_member("localhost:3000", &member, "member").await; + seed_relay_member(&relay_authority(), &member, "member").await; let response = invite_post(&member, "/api/invites", "{}").await; assert_eq!(response.status(), reqwest::StatusCode::FORBIDDEN); @@ -791,10 +796,10 @@ async fn test_auth_event_kind_rejected() { /// NIP-11 max_subscriptions must be enforced; (limit+1)th REQ gets CLOSED. /// -/// The relay's MAX_SUBSCRIPTIONS is 1024. Opening 1024 subs in a test is slow, -/// so we open a smaller batch and verify the NIP-11 advertised limit matches -/// the actual enforcement constant. The full-limit test is covered by the -/// NIP-11 assertion below (which verifies the advertised value is 1024). +/// This is a protocol-cap test, not an admission-throughput test. Open one REQ +/// at a time and wait out any shared fixed-window quota before retrying a REQ +/// rejected specifically as `rate-limited`, so production admission remains +/// enabled while the test deterministically reaches the independent 1024 cap. #[tokio::test] #[ignore] async fn test_subscription_limit_enforced() { @@ -802,60 +807,75 @@ async fn test_subscription_limit_enforced() { let keys = Keys::generate(); let mut client = BuzzTestClient::connect(&url, &keys).await.expect("connect"); - // Open 1024 subscriptions (the relay's MAX_SUBSCRIPTIONS). for i in 0..1024 { let sid = format!("limit-sub-{i}"); - let filter = Filter::new().kind(Kind::Custom(9)); - client - .subscribe(&sid, vec![filter]) - .await - .expect("subscribe"); - // Drain EOSE to avoid buffer buildup. - client - .collect_until_eose(&sid, Duration::from_secs(5)) - .await - .expect("EOSE"); + let filter = Filter::new().kind(Kind::Custom(49_999)); + subscribe_until_eose(&mut client, &sid, filter).await; } let overflow_sid = sub_id("overflow"); - // Use a kind that no other test writes, so we don't receive stale events. - let filter = Filter::new().kind(Kind::Custom(49999)); - client - .subscribe(&overflow_sid, vec![filter]) - .await - .expect("send REQ"); - - // Drain EOSE and stale events from the 100 earlier subscriptions - // until we receive the CLOSED for the overflow subscription. - let msg = loop { - let m = client - .recv_event(Duration::from_secs(5)) + let filter = Filter::new().kind(Kind::Custom(49_999)); + loop { + client + .subscribe(&overflow_sid, vec![filter.clone()]) .await - .expect("recv CLOSED (or timeout)"); - match &m { - RelayMessage::Eose { .. } => continue, - RelayMessage::Event { .. } => continue, // stale event from earlier subs - _ => break m, - } - }; + .expect("send overflow REQ"); - match msg { - RelayMessage::Closed { - subscription_id, - message, - } => { - assert_eq!(subscription_id, overflow_sid); - assert!( - message.to_lowercase().contains("too many"), - "Expected 'too many' in CLOSED message, got: {message}" - ); + match client + .recv_event(Duration::from_secs(6)) + .await + .expect("recv overflow CLOSED") + { + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id == overflow_sid && message.starts_with("rate-limited:") => { + tokio::time::sleep(Duration::from_secs(5)).await; + } + RelayMessage::Closed { + subscription_id, + message, + } => { + assert_eq!(subscription_id, overflow_sid); + assert!( + message.to_lowercase().contains("too many"), + "Expected 'too many' in CLOSED message, got: {message}" + ); + break; + } + other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } - other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } client.disconnect().await.expect("disconnect"); } +async fn subscribe_until_eose(client: &mut BuzzTestClient, sid: &str, filter: Filter) { + loop { + client + .subscribe(sid, vec![filter.clone()]) + .await + .expect("subscribe"); + match client + .recv_event(Duration::from_secs(6)) + .await + .expect("EOSE or rate-limit CLOSED") + { + RelayMessage::Eose { subscription_id } => { + assert_eq!(subscription_id, sid); + return; + } + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id == sid && message.starts_with("rate-limited:") => { + tokio::time::sleep(Duration::from_secs(5)).await; + } + other => panic!("unexpected response while opening {sid}: {other:?}"), + } + } +} + #[tokio::test] #[ignore] async fn test_nip11_relay_info() { diff --git a/deploy/local/build-and-deploy.sh b/deploy/local/build-and-deploy.sh index 2251ce10a3..4dce0fdc36 100755 --- a/deploy/local/build-and-deploy.sh +++ b/deploy/local/build-and-deploy.sh @@ -103,14 +103,17 @@ if [ "$helm_rc" != 0 ]; then fi # ── 4. verify 3/3 Ready ─────────────────────────────────────────────────────── -# Select the relay by its explicit component label. Optional worker/sidecar -# Deployments share the release instance and must never become the rollout target. -DEPLOYMENTS=$(kubectl -n "$NS" get deploy \ - -l "app.kubernetes.io/instance=$RELEASE,app.kubernetes.io/component=relay" \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') -[ "$(printf '%s\n' "$DEPLOYMENTS" | sed '/^$/d' | wc -l | tr -d ' ')" = "1" ] || \ - die "expected exactly one relay Deployment, got: ${DEPLOYMENTS:-}" -DEPLOY=$(printf '%s\n' "$DEPLOYMENTS" | sed '/^$/d') +# Find the relay Deployment: everything under this release named "buzz" except +# the bundled "*-minio" Deployment. (The chart fullname collapses +# "-" to "" when the release name already contains the +# chart name, so the name isn't always "-buzz".) +DEPLOY="" +for d in $(kubectl -n "$NS" get deploy -l "app.kubernetes.io/instance=$RELEASE" \ + -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do + case "$d" in *-minio) continue;; esac + DEPLOY="$d"; break +done +[ -n "$DEPLOY" ] || die "could not locate the relay Deployment" log "waiting for $REPLICAS relay pods Ready (deployment: $DEPLOY)" kubectl -n "$NS" rollout status deployment/"$DEPLOY" --timeout=4m | tee "$EVID/rollout.txt" kubectl -n "$NS" get pods -o wide | tee "$EVID/pods.txt" @@ -126,8 +129,9 @@ log "probing /_readiness on each relay pod individually" : > "$EVID/readiness.txt" FAIL=0 RELAY_PODS=$(kubectl -n "$NS" get pods \ - -l "app.kubernetes.io/instance=$RELEASE,app.kubernetes.io/component=relay" \ - -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') + -l "app.kubernetes.io/name=buzz,app.kubernetes.io/instance=$RELEASE" \ + -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.app\.kubernetes\.io/component}{"\n"}{end}' \ + | awk '$2 != "minio" && $2 != "minio-init" {print $1}') for pod in $RELAY_PODS; do body=$(kubectl -n "$NS" exec "$pod" -- \ sh -c 'curl -sS --max-time 5 http://127.0.0.1:8080/_readiness' 2>/dev/null || echo '') From b69bd76029d4e95a4da18f64721c526d9c2f2b09 Mon Sep 17 00:00:00 2001 From: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 10:54:53 -0700 Subject: [PATCH 08/51] fix(deletion): make community erasure resumable Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Co-authored-by: Kalvin Chau Signed-off-by: Kalvin Chau --- crates/buzz-db/src/deletion.rs | 647 ++++++++++++++---- crates/buzz-db/src/lib.rs | 7 +- crates/buzz-db/src/migration.rs | 2 + crates/buzz-deletion/src/lib.rs | 382 ++++++++--- crates/buzz-media/src/error.rs | 24 +- crates/buzz-media/src/storage.rs | 32 +- crates/buzz-relay/src/api/git/transport.rs | 13 +- crates/buzz-relay/src/api/media.rs | 41 +- crates/buzz-relay/src/handlers/event.rs | 108 +-- crates/buzz-relay/src/handlers/ingest.rs | 5 +- crates/buzz-relay/src/main.rs | 26 +- crates/buzz-relay/src/router.rs | 2 +- deploy/charts/buzz/README.md | 32 +- .../templates/deletion-worker-deployment.yaml | 1 + .../buzz/tests/deletion_worker_test.yaml | 7 +- deploy/charts/buzz/values.schema.json | 18 + deploy/charts/buzz/values.yaml | 2 + migrations/0028_community_deletion.sql | 8 +- schema/schema.sql | 2 +- 19 files changed, 1066 insertions(+), 293 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 71070874e8..c9df6d1471 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -283,6 +283,9 @@ pub struct StorageManifest { pub version: i32, /// Every tenant-owned key that will be removed, sorted. pub tenant_keys: Vec, + /// Immutable observations used to detect replacement under an approved key. + #[serde(default)] + pub tenant_objects: Vec, /// Git pointer keys among tenant keys. pub git_pointer_keys: Vec, /// Media sidecar keys among tenant keys. @@ -298,6 +301,17 @@ pub struct StorageManifest { pub unsupported_version_keys: Vec, } +/// Frozen observation of one tenant-owned object binding. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] +pub struct StorageObject { + /// Exact bucket key. + pub key: String, + /// Current object size. + pub size: u64, + /// Opaque entity tag used to detect object replacement. + pub e_tag: Option, +} + /// Full frozen inventory approved at the destructive boundary. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct FrozenInventory { @@ -421,6 +435,24 @@ pub struct ServingWriteLease { pub lease_until: DateTime, } +/// Validate the minimum catalog contract used by serving-path fences. +pub const REQUIRED_SERVING_TABLES: &[&str] = &[ + "communities", + "community_serving_write_leases", + "community_deletion_requests", +]; + +/// Bounded-cardinality operational snapshot for the hot serving-lease table. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ServingLeaseStats { + /// Unexpired serving-write leases. + pub active: i64, + /// Expired rows awaiting cleanup. + pub expired: i64, + /// PostgreSQL's estimated dead tuples for the lease table. + pub dead_tuples: i64, +} + /// PostgreSQL deletion adapter. Clone is cheap. #[derive(Clone)] pub struct DeletionStore { @@ -581,23 +613,128 @@ impl DeletionStore { }) } - /// Validate the live scoped-table and write-fence catalog before serving. + /// Validate the minimum deletion-fence catalog required by relay serving. /// - /// This uses the same exact-set checks as deletion inventory but does not - /// require a community row. Schema drift therefore fails startup/readiness - /// even before an operator submits a deletion request. - pub async fn validate_catalog(&self) -> Result<()> { - let migration_version: i64 = sqlx::query_scalar( - "SELECT COALESCE(max(version), 0) FROM _sqlx_migrations WHERE success", + /// Serving binaries accept newer additive migrations for rolling upgrades + /// and rollback. Destructive inventory still calls [`Self::validate_catalog`] + /// and requires exact migration/table equality. + pub async fn validate_serving_catalog(&self) -> Result<()> { + let migration_version = self.live_migration_version().await?; + validate_serving_migration_version(migration_version)?; + + let runtime_columns = sqlx::query( + "SELECT attname, format_type(atttypid, atttypmod) AS type_name, attnotnull \ + FROM pg_attribute WHERE attrelid = 'communities'::regclass \ + AND attname IN ('deletion_state', 'deletion_fence_generation', 'deleted_at') \ + AND NOT attisdropped ORDER BY attname", ) - .fetch_one(&self.pool) + .fetch_all(&self.pool) .await?; - if migration_version != EXPECTED_MIGRATION_VERSION { + let column_contract = runtime_columns + .iter() + .map(|row| { + Ok::<_, DbError>(( + row.try_get::("attname")?, + row.try_get::("type_name")?, + row.try_get::("attnotnull")?, + )) + }) + .collect::>>()?; + let expected_columns = BTreeSet::from([ + ( + "deleted_at".to_string(), + "timestamp with time zone".to_string(), + false, + ), + ( + "deletion_fence_generation".to_string(), + "bigint".to_string(), + true, + ), + ("deletion_state".to_string(), "text".to_string(), true), + ]); + if column_contract != expected_columns { + return Err(DbError::DeletionSafety( + "community serving fence columns are missing or incompatible".to_string(), + )); + } + + let required_tables = REQUIRED_SERVING_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + let required_table_names = REQUIRED_SERVING_TABLES + .iter() + .map(ToString::to_string) + .collect::>(); + let live_tables: BTreeSet = sqlx::query_scalar( + "SELECT table_name FROM information_schema.tables \ + WHERE table_schema = 'public' AND table_name = ANY($1) \ + ORDER BY table_name", + ) + .bind(&required_table_names) + .fetch_all(&self.pool) + .await? + .into_iter() + .collect(); + if live_tables != required_tables { return Err(DbError::DeletionSafety(format!( - "community deletion schema migration drift: expected {EXPECTED_MIGRATION_VERSION}, got {migration_version}" + "community serving fence tables missing: {}", + required_tables + .difference(&live_tables) + .cloned() + .collect::>() + .join(",") ))); } + let required_fences = EXPECTED_SCOPED_TABLES + .iter() + .copied() + .map(str::to_owned) + .collect::>(); + let live_fences = self.live_fenced_tables().await?; + let missing_fences = required_fences + .difference(&live_fences) + .cloned() + .collect::>(); + if !missing_fences.is_empty() { + return Err(DbError::DeletionSafety(format!( + "community serving write fences missing: {}", + missing_fences.join(",") + ))); + } + + let required_objects_present: bool = sqlx::query_scalar( + "SELECT to_regprocedure('community_deletion_lock_key(uuid)') IS NOT NULL \ + AND to_regprocedure('assert_community_write_allowed(uuid)') IS NOT NULL \ + AND to_regprocedure('enforce_community_write_fence()') IS NOT NULL \ + AND EXISTS (SELECT 1 FROM pg_trigger t \ + JOIN pg_class c ON c.oid = t.tgrelid \ + JOIN pg_proc p ON p.oid = t.tgfoid \ + WHERE c.relname = 'communities' \ + AND p.proname = 'enforce_community_tombstone' \ + AND NOT t.tgisinternal AND t.tgenabled = 'O')", + ) + .fetch_one(&self.pool) + .await?; + if !required_objects_present { + return Err(DbError::DeletionSafety( + "community serving fence functions or tombstone trigger are missing".to_string(), + )); + } + Ok(()) + } + + /// Validate the exact live scoped-table and write-fence catalog for destruction. + /// + /// Unlike relay serving compatibility, this intentionally rejects newer + /// migrations and unknown tenant tables until the deletion manifest changes. + pub async fn validate_catalog(&self) -> Result<()> { + let migration_version = self.live_migration_version().await?; + validate_destructive_migration_version(migration_version)?; + let expected = EXPECTED_SCOPED_TABLES .iter() .copied() @@ -758,6 +895,9 @@ impl DeletionStore { owner: &str, lease_duration: Duration, ) -> Result> { + // Claim is the destructive worker boundary: unlike serving readiness, + // execution refuses any newer/unknown catalog until this engine knows it. + self.validate_catalog().await?; let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); let row = sqlx::query( r#" @@ -890,7 +1030,57 @@ impl DeletionStore { Ok(()) } - /// Acquire the universal durable fence and move approved → fenced. + /// Persist quiescing intent before waiting for active serving leases. + /// + /// This is the irreversible fail-closed point: a request intentionally has + /// no automatic unquiesce/unblock transition after operator approval. + /// + /// The transition takes the same exclusive advisory lock as serving lease + /// acquisition, so after commit no newer external effect can be admitted. + /// Already-acquired leases remain verifiable/releasable but cannot renew. + pub async fn begin_quiescing(&self, token: &LeaseToken) -> Result<()> { + let mut tx = self.pool.begin().await?; + verify_lease(&mut tx, token, DeletionStage::Approved).await?; + sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await?; + let generation: i64 = sqlx::query_scalar( + "SELECT deletion_fence_generation FROM communities WHERE id = $1 FOR UPDATE", + ) + .bind(token.community_id.as_uuid()) + .fetch_one(&mut *tx) + .await?; + set_executor_gucs(&mut tx, token.community_id, generation).await?; + let affected = sqlx::query( + "UPDATE communities SET deletion_state = 'quiescing', \ + archived_at = COALESCE(archived_at, now()) \ + WHERE id = $1 AND deletion_state IN ('active', 'quiescing') \ + AND deleted_at IS NULL", + ) + .bind(token.community_id.as_uuid()) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} cannot enter quiescing", + token.community_id + ))); + } + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Approved, + "quiesce_serving_writes", + serde_json::json!({"community_state": "quiescing"}), + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Acquire the universal durable fence after all pre-quiesce serving leases drain. pub async fn fence(&self, token: &LeaseToken) -> Result { let mut tx = self.pool.begin().await?; verify_lease(&mut tx, token, DeletionStage::Approved).await?; @@ -928,7 +1118,7 @@ impl DeletionStore { let affected = sqlx::query( "UPDATE communities SET deletion_state = 'fenced', \ deletion_fence_generation = $2, archived_at = COALESCE(archived_at, now()) \ - WHERE id = $1 AND deletion_state = 'active'", + WHERE id = $1 AND deletion_state = 'quiescing'", ) .bind(token.community_id.as_uuid()) .bind(generation) @@ -937,7 +1127,7 @@ impl DeletionStore { .rows_affected(); if affected != 1 { return Err(DbError::DeletionSafety(format!( - "community {} is no longer active while fencing", + "community {} is no longer quiescing while fencing", token.community_id ))); } @@ -1047,6 +1237,57 @@ impl DeletionStore { Ok(()) } + /// Record one completed object-binding deletion checkpoint. + pub async fn checkpoint_storage_object_removed( + &self, + token: &LeaseToken, + key: &str, + already_missing: bool, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Drained, generation).await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Drained, + &storage_checkpoint_key(key), + serde_json::json!({"key": key, "already_missing": already_missing}), + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Return object keys already durably removed in the Drained stage. + pub async fn completed_storage_object_keys( + &self, + token: &LeaseToken, + ) -> Result> { + let rows: Vec = sqlx::query_scalar( + "SELECT detail FROM community_deletion_checkpoints \ + WHERE request_id = $1 AND stage = 'drained' AND status = 'completed' \ + AND unit_key LIKE 'object:%' ORDER BY sequence", + ) + .bind(token.request_id) + .fetch_all(&self.pool) + .await?; + rows.into_iter() + .map(|detail| { + detail + .get("key") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "malformed storage checkpoint for deletion {}", + token.request_id + )) + }) + }) + .collect() + } + /// Mark storage binding removal after adapter verification. pub async fn mark_bindings_removed( &self, @@ -1077,8 +1318,8 @@ impl DeletionStore { set_executor_gucs(&mut tx, token.community_id, generation).await?; let mut deleted = BTreeMap::new(); - // Deleting channels cascades some child tables. Alphabetical execution - // plus per-table WHERE is safe and idempotent; later units simply see 0. + // The order is child-before-parent/FK-safe, not alphabetical. Cascades + // can make later units observe zero rows; each scoped WHERE stays idempotent. for table in PURGE_SCOPED_TABLES { let sql = format!("DELETE FROM {table} WHERE community_id = $1"); let affected = sqlx::query(AssertSqlSafe(sql)) @@ -1097,7 +1338,7 @@ impl DeletionStore { .await?; } - sqlx::query( + let affected = sqlx::query( "UPDATE communities SET deletion_state = 'tombstone', \ deleted_at = COALESCE(deleted_at, now()), \ archived_at = COALESCE(archived_at, now()), \ @@ -1108,7 +1349,14 @@ impl DeletionStore { .bind(token.community_id.as_uuid()) .bind(generation) .execute(&mut *tx) - .await?; + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "community {} tombstone update affected {affected} rows", + token.community_id + ))); + } advance_request_tx( &mut tx, token, @@ -1378,8 +1626,11 @@ impl DeletionStore { Ok(lease) } - /// Renew an external side-effect lease only while ownership and the - /// community's active lifecycle are still current. + /// Renew an external side-effect lease only while the community is active. + /// + /// Quiescing rejects new acquisition and renewal. A pre-quiesce caller may + /// still verify/release its unexpired lease, but a long operation is + /// cancelled when its next heartbeat observes quiescing. pub async fn renew_serving_write_lease( &self, lease: &mut ServingWriteLease, @@ -1438,8 +1689,10 @@ impl DeletionStore { Ok(deleted == 1) } - /// Check that an external side-effect lease and the community's active - /// lifecycle are still current. + /// Check that an external side-effect lease remains current for finalization. + /// + /// A lease admitted before quiescing may complete/release, but cannot renew; + /// this preserves an accurate bounded drain without admitting new work. pub async fn verify_serving_write_lease(&self, lease: &ServingWriteLease) -> Result<()> { let mut tx = self.pool.begin().await?; sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") @@ -1453,7 +1706,7 @@ impl DeletionStore { AND lease.generation = $4 AND lease.fence_generation = $5 \ AND lease.lease_until >= now() \ AND community.deleted_at IS NULL \ - AND community.deletion_state = 'active' \ + AND community.deletion_state IN ('active', 'quiescing') \ AND community.deletion_fence_generation = lease.fence_generation)", ) .bind(lease.id) @@ -1474,6 +1727,41 @@ impl DeletionStore { } } + /// Delete expired serving leases in a bounded batch. + pub async fn reap_expired_serving_write_leases(&self, limit: i64) -> Result { + let affected = sqlx::query( + "WITH expired AS ( \ + SELECT id FROM community_serving_write_leases \ + WHERE lease_until < now() ORDER BY lease_until LIMIT $1 \ + FOR UPDATE SKIP LOCKED \ + ) DELETE FROM community_serving_write_leases lease \ + USING expired WHERE lease.id = expired.id", + ) + .bind(limit.clamp(1, 10_000)) + .execute(&self.pool) + .await? + .rows_affected(); + Ok(affected) + } + + /// Return serving-lease counts and dead-tuple estimate for observability. + pub async fn serving_lease_stats(&self) -> Result { + let row = sqlx::query( + "SELECT count(*) FILTER (WHERE lease_until >= now())::BIGINT AS active, \ + count(*) FILTER (WHERE lease_until < now())::BIGINT AS expired, \ + COALESCE((SELECT n_dead_tup::BIGINT FROM pg_stat_user_tables \ + WHERE relname = 'community_serving_write_leases'), 0) AS dead_tuples \ + FROM community_serving_write_leases", + ) + .fetch_one(&self.pool) + .await?; + Ok(ServingLeaseStats { + active: row.try_get("active")?, + expired: row.try_get("expired")?, + dead_tuples: row.try_get("dead_tuples")?, + }) + } + /// Whether a community remains active and serving-write eligible. pub async fn is_serving_active(&self, community: CommunityId) -> Result { sqlx::query_scalar( @@ -1503,6 +1791,13 @@ impl DeletionStore { Ok(()) } + async fn live_migration_version(&self) -> Result { + sqlx::query_scalar("SELECT COALESCE(max(version), 0) FROM _sqlx_migrations WHERE success") + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + async fn live_scoped_tables(&self) -> Result> { let rows: Vec = sqlx::query_scalar( r#" @@ -1551,14 +1846,67 @@ impl DeletionStore { } } +fn validate_destructive_migration_version(migration_version: i64) -> Result<()> { + if migration_version != EXPECTED_MIGRATION_VERSION { + Err(DbError::DeletionSafety(format!( + "community deletion schema migration drift: expected {EXPECTED_MIGRATION_VERSION}, got {migration_version}" + ))) + } else { + Ok(()) + } +} + +fn validate_serving_migration_version(migration_version: i64) -> Result<()> { + if migration_version < EXPECTED_MIGRATION_VERSION { + Err(DbError::DeletionSafety(format!( + "community serving fence migration is too old: require at least {EXPECTED_MIGRATION_VERSION}, got {migration_version}" + ))) + } else { + Ok(()) + } +} + /// Fail closed when storage inventory reports unknown or unsupported data. pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { - if manifest.version != 1 { + if manifest.version != 2 { return Err(DbError::DeletionSafety(format!( "unsupported storage manifest version {}", manifest.version ))); } + let object_keys = manifest + .tenant_objects + .iter() + .map(|object| object.key.as_str()) + .collect::>(); + let tenant_keys = manifest + .tenant_keys + .iter() + .map(String::as_str) + .collect::>(); + if object_keys != tenant_keys { + return Err(DbError::DeletionSafety( + "storage manifest object observations do not exactly match tenant keys".to_string(), + )); + } + if manifest + .tenant_objects + .iter() + .any(|object| object.e_tag.is_none()) + { + return Err(DbError::DeletionSafety( + "storage manifest contains a tenant object without ETag".to_string(), + )); + } + if manifest + .tenant_objects + .windows(2) + .any(|pair| pair[0].key >= pair[1].key) + { + return Err(DbError::DeletionSafety( + "storage manifest tenant objects are not strictly sorted".to_string(), + )); + } if !manifest.unknown_keys.is_empty() { return Err(DbError::DeletionSafety(format!( "unknown object-store keys block deletion: {}", @@ -1736,6 +2084,10 @@ async fn checkpoint_failed_tx( Ok(()) } +fn storage_checkpoint_key(key: &str) -> String { + format!("object:{}", hex::encode(Sha256::digest(key.as_bytes()))) +} + fn serialize_community_id( community: &CommunityId, serializer: S, @@ -1775,6 +2127,11 @@ fn row_to_request(row: sqlx::postgres::PgRow) -> Result { }) } +/// Return whether an error is the deletion store's typed ownership-loss class. +pub fn is_stale_deletion_lease(error: &DbError) -> bool { + matches!(error, DbError::AccessDenied(message) if message.starts_with("stale deletion lease ") || message.starts_with("stale lease or fencing generation for deletion ")) +} + fn stale_lease_error(token: &LeaseToken) -> DbError { DbError::AccessDenied(format!( "stale deletion lease {} owner {:?} generation {}", @@ -1808,8 +2165,9 @@ mod tests { fn storage_manifest() -> StorageManifest { StorageManifest { - version: 1, + version: 2, tenant_keys: Vec::new(), + tenant_objects: Vec::new(), git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), @@ -1848,6 +2206,56 @@ mod tests { assert!(!DeletionStage::RetentionPending.runnable()); } + #[test] + fn stale_lease_classifier_does_not_swallow_other_access_denials() { + let stale = stale_lease_error(&LeaseToken { + request_id: Uuid::new_v4(), + owner: "owner".to_string(), + generation: 1, + community_id: CommunityId::from_uuid(Uuid::new_v4()), + fence_generation: None, + }); + assert!(is_stale_deletion_lease(&stale)); + assert!(!is_stale_deletion_lease(&DbError::AccessDenied( + "ordinary authorization failure".to_string() + ))); + } + + #[test] + fn serving_catalog_accepts_future_migrations_but_destructive_catalog_does_not() { + assert!(validate_serving_migration_version(EXPECTED_MIGRATION_VERSION).is_ok()); + assert!(validate_serving_migration_version(EXPECTED_MIGRATION_VERSION + 1).is_ok()); + assert!(validate_serving_migration_version(EXPECTED_MIGRATION_VERSION - 1).is_err()); + assert!(validate_destructive_migration_version(EXPECTED_MIGRATION_VERSION).is_ok()); + assert!(validate_destructive_migration_version(EXPECTED_MIGRATION_VERSION + 1).is_err()); + } + + #[test] + fn storage_manifest_rejects_missing_identity_observation() { + let mut manifest = storage_manifest(); + manifest.tenant_keys.push("tenant/key".to_string()); + assert!(validate_storage_manifest(&manifest).is_err()); + } + + #[test] + fn storage_manifest_rejects_missing_object_etag() { + let mut manifest = storage_manifest(); + manifest.tenant_keys.push("tenant/key".to_string()); + manifest.tenant_objects.push(StorageObject { + key: "tenant/key".to_string(), + size: 1, + e_tag: None, + }); + assert!(validate_storage_manifest(&manifest).is_err()); + } + + #[test] + fn object_checkpoint_keys_are_bounded_and_stable() { + let key = "x".repeat(4096); + assert_eq!(storage_checkpoint_key(&key), storage_checkpoint_key(&key)); + assert_eq!(storage_checkpoint_key(&key).len(), "object:".len() + 64); + } + #[test] fn unknown_storage_data_fails_closed() { let mut manifest = storage_manifest(); @@ -1933,8 +2341,9 @@ mod postgres_tests { .await .expect("schema inventory"), storage: StorageManifest { - version: 1, + version: 2, tenant_keys: Vec::new(), + tenant_objects: Vec::new(), git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), @@ -2011,6 +2420,7 @@ mod postgres_tests { "stale lease must reject" ); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); let generation = store.fence(&claim.lease).await.expect("fence"); let mut wrong_fence = claim.lease.clone(); wrong_fence.fence_generation = Some(generation + 1); @@ -2048,7 +2458,10 @@ mod postgres_tests { let store_for_fence = store.clone(); let lease = claim.lease.clone(); - let fencing = tokio::spawn(async move { store_for_fence.fence(&lease).await }); + let fencing = tokio::spawn(async move { + store_for_fence.begin_quiescing(&lease).await?; + store_for_fence.fence(&lease).await + }); tokio::time::sleep(Duration::from_millis(50)).await; assert!( !fencing.is_finished(), @@ -2070,7 +2483,7 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn active_serving_lease_retries_fence_without_lifecycle_change() { + async fn quiescing_rejects_new_and_renewed_leases_before_fence() { let (db, store) = store().await; let (request, _) = inventoried_request(&db, &store).await; store @@ -2082,7 +2495,7 @@ mod postgres_tests { .await .expect("claim") .expect("won claim"); - let serving = store + let mut serving = store .acquire_serving_write_lease( request.community_id, "test_external", @@ -2092,37 +2505,45 @@ mod postgres_tests { .await .expect("serving lease"); - let error = store - .fence(&claim.lease) + store + .begin_quiescing(&claim.lease) .await - .expect_err("active serving lease must make fence retry"); + .expect("persist quiescing"); + assert!(matches!( + store + .acquire_serving_write_lease( + request.community_id, + "late_external", + "late-owner", + DEFAULT_LEASE_DURATION, + ) + .await, + Err(DbError::AccessDenied(_)) + )); + assert!(store.verify_serving_write_lease(&serving).await.is_ok()); + assert!(matches!( + store + .renew_serving_write_lease(&mut serving, DEFAULT_LEASE_DURATION) + .await, + Err(DbError::AccessDenied(_)) + )); assert!(matches!( - error, - DbError::ServingWritesNotDrained { + store.fence(&claim.lease).await, + Err(DbError::ServingWritesNotDrained { active_count: 1, - ref operations, .. - } if operations == &["test_external"] + }) )); - let unchanged = store.get(request.id).await.expect("request after retry"); - assert_eq!(unchanged.stage, DeletionStage::Approved); - assert_eq!(unchanged.fence_generation, None); - assert!(store - .is_serving_active(request.community_id) - .await - .expect("community remains active")); - assert!(store .release_serving_write_lease(&serving) .await .expect("release")); - let generation = store.fence(&claim.lease).await.expect("retry fence"); - assert_eq!(generation, 1); + assert_eq!(store.fence(&claim.lease).await.expect("fence"), 1); } #[tokio::test] #[ignore = "requires Postgres"] - async fn serving_lease_acquisition_and_fence_have_exactly_one_winner() { + async fn sustained_admission_cannot_starve_fence_after_quiescing() { let (db, store) = store().await; let (request, _) = inventoried_request(&db, &store).await; store @@ -2134,104 +2555,55 @@ mod postgres_tests { .await .expect("claim") .expect("won claim"); - - let mut lock_holder = db.pool.acquire().await.expect("lock connection"); - sqlx::query("BEGIN") - .execute(&mut *lock_holder) - .await - .expect("begin lock transaction"); - sqlx::query("SELECT pg_advisory_xact_lock(community_deletion_lock_key($1))") - .bind(request.community_id.as_uuid()) - .execute(&mut *lock_holder) - .await - .expect("hold ordering lock"); - - let acquire_store = store.clone(); - let community = request.community_id; - let acquire = tokio::spawn(async move { - acquire_store - .acquire_serving_write_lease( - community, - "race_acquire", - "race-owner", - DEFAULT_LEASE_DURATION, - ) - .await - }); - let fence_store = store.clone(); - let fence_token = claim.lease.clone(); - let fence = tokio::spawn(async move { fence_store.fence(&fence_token).await }); - sqlx::query("COMMIT") - .execute(&mut *lock_holder) - .await - .expect("release ordering lock"); - - let lease_result = acquire.await.expect("acquire task"); - let fence_result = fence.await.expect("fence task"); - match (lease_result, fence_result) { - (Ok(lease), Err(DbError::ServingWritesNotDrained { .. })) => { - assert!(store - .is_serving_active(request.community_id) - .await - .expect("active after lease wins")); - assert!(store - .release_serving_write_lease(&lease) - .await - .expect("release winner")); - } - (Err(DbError::AccessDenied(_)), Ok(_)) => { - assert!(!store - .is_serving_active(request.community_id) - .await - .expect("fenced after deletion wins")); - } - other => panic!("invalid acquisition/fence race outcome: {other:?}"), + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + + for attempt in 0..100 { + assert!(matches!( + store + .acquire_serving_write_lease( + request.community_id, + "sustained_admission", + &format!("owner-{attempt}"), + DEFAULT_LEASE_DURATION, + ) + .await, + Err(DbError::AccessDenied(_)) + )); } + assert_eq!(store.fence(&claim.lease).await.expect("fence"), 1); } #[tokio::test] #[ignore = "requires Postgres"] - async fn external_serving_lease_blocks_drain_without_holding_a_pool_connection() { + async fn serving_lease_reaper_is_bounded_and_reports_stats() { let (db, store) = store().await; - let (request, _) = inventoried_request(&db, &store).await; - store - .approve(request.id, "approver", None) - .await - .expect("approve"); - let claim = store - .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) - .await - .expect("claim") - .expect("won claim"); - let serving = store - .acquire_serving_write_lease( - request.community_id, - "test_external", - "test-owner", - DEFAULT_LEASE_DURATION, - ) - .await - .expect("serving lease"); - assert!(matches!( - store.fence(&claim.lease).await, - Err(DbError::ServingWritesNotDrained { .. }) - )); - assert!(store - .release_serving_write_lease(&serving) - .await - .expect("release")); - let generation = store - .fence(&claim.lease) - .await - .expect("fence after release"); - let token = LeaseToken { - fence_generation: Some(generation), - ..claim.lease - }; - store - .mark_drained(&token) + let host = format!("lease-reaper-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) .await - .expect("drain immediately after successful fence"); + .expect("community") + .id; + for owner in ["expired-a", "expired-b", "expired-c"] { + let lease = store + .acquire_serving_write_lease( + community, + "reaper_test", + owner, + Duration::from_secs(1), + ) + .await + .expect("lease"); + sqlx::query("UPDATE community_serving_write_leases SET lease_until = now() - interval '1 second' WHERE id = $1") + .bind(lease.id) + .execute(&db.pool) + .await + .expect("expire lease"); + } + let before = store.serving_lease_stats().await.expect("stats before"); + assert!(before.expired >= 3); + assert_eq!(store.reap_expired_serving_write_leases(2).await.unwrap(), 2); + let after = store.serving_lease_stats().await.expect("stats after"); + assert_eq!(after.expired, before.expired - 2); } #[tokio::test] @@ -2249,6 +2621,7 @@ mod postgres_tests { .await .expect("claim") .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); let generation = store.fence(&claim.lease).await.expect("fence"); let token = LeaseToken { fence_generation: Some(generation), diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index cdb01b6c8c..6b9a5afd98 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -1020,7 +1020,12 @@ impl Db { sqlx::query("SELECT 1").execute(&self.pool).await.is_ok() } - /// Validate the live community-deletion tenant catalog and write fences. + /// Validate the minimum deletion fence catalog required by serving paths. + pub async fn validate_deletion_serving_catalog(&self) -> Result<()> { + self.deletion_store().validate_serving_catalog().await + } + + /// Validate the exact live community-deletion tenant catalog for destruction. pub async fn validate_deletion_catalog(&self) -> Result<()> { self.deletion_store().validate_catalog().await } diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 6aa481d55a..cbb8e0e062 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -1120,6 +1120,8 @@ mod tests { assert!(deletion.contains("community_write_fence_excluded_table")); assert!(deletion.contains("CREATE FUNCTION enforce_community_tombstone")); assert!(deletion.contains("community tombstones are permanent")); + assert!(deletion.contains("SET LOCAL lock_timeout = '5s'")); + assert!(deletion.contains("'active', 'quiescing', 'fenced', 'tombstone'")); assert!(deletion.contains("_operator_global_tables")); assert!(deletion.contains("'submitted', 'inventoried', 'approved', 'fenced', 'drained'")); >>>>>>> d04e4b503 (feat: add durable community deletion worker) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 372eb13686..df10e60c6b 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -9,7 +9,7 @@ use std::time::Duration; use anyhow::{Context, Result}; use buzz_db::deletion::{ ClaimedDeletion, DeletionRequest, DeletionStage, DeletionStore, FrozenInventory, LeaseToken, - StorageManifest, DEFAULT_LEASE_DURATION, + StorageManifest, StorageObject, DEFAULT_LEASE_DURATION, }; use buzz_db::{Db, DbConfig}; use buzz_media::{deletion_inventory, CurrentObjectVersion, MediaStorage}; @@ -22,6 +22,7 @@ const DEFAULT_STORAGE_OBJECT_CAP: u64 = 1_000_000; const WORKER_IDLE_POLL: Duration = Duration::from_secs(5); const RETRY_DELAY: Duration = Duration::from_secs(30); const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10); +const STORAGE_DELETE_BATCH_SIZE: usize = 100; #[cfg(test)] static TEST_HEARTBEAT_INTERVAL_MS: AtomicU64 = AtomicU64::new(0); @@ -111,6 +112,14 @@ impl ServingWriteGuard { error.downcast_ref::().is_some() } + /// Whether a serving-write acquisition failed because the tenant is fenced. + pub fn acquisition_is_fenced(error: &anyhow::Error) -> bool { + matches!( + error.downcast_ref::(), + Some(buzz_db::DbError::AccessDenied(_)) + ) + } + /// Signal fired if the background lease heartbeat fails. pub fn lost(&self) -> CancellationToken { self.lost.clone() @@ -146,6 +155,11 @@ impl Drop for ServingWriteGuard { } /// Acquire a serving-side external-effect lease without holding a pool connection. +/// +/// A separate short database lease per effect is intentional: it is the only +/// durable proof that deletion can drain S3/Redis/push work across replicas. +/// PostgreSQL lease-table churn is reaped and exported by the relay pool-metrics +/// task; operators should watch the deletion lease gauges documented by Helm. pub async fn acquire_serving_write( db: &Db, community: buzz_core::CommunityId, @@ -503,6 +517,18 @@ fn storage_taxonomy_matches(approved: &StorageManifest, live: &StorageManifest) && approved.unsupported_version_keys == live.unsupported_version_keys } +fn missing_binding_is_resumable(completed_count: usize) -> bool { + completed_count > 0 +} + +fn storage_object_matches( + expected: &StorageObject, + observed_size: u64, + observed_e_tag: Option<&str>, +) -> bool { + expected.size == observed_size && observed_e_tag == expected.e_tag.as_deref() +} + async fn build_inventory( services: &Services, request: &DeletionRequest, @@ -515,32 +541,52 @@ async fn build_inventory( Ok(FrozenInventory { schema, storage }) } -async fn build_storage_manifest( +async fn build_storage_manifest_from_objects( services: &Services, request: &DeletionRequest, + objects: Vec, ) -> Result { - let max_objects = storage_object_cap(); - let objects = services.media.list_all_for_deletion(max_objects).await?; let bucket = deletion_inventory( *request.community_id.as_uuid(), objects .iter() .map(|object| (object.key.clone(), object.size)), ); + let tenant_keys = bucket.tenant_keys(); + let tenant_key_set = tenant_keys.iter().collect::>(); + let tenant_objects = objects + .into_iter() + .filter(|object| tenant_key_set.contains(&object.key)) + .map(|object| StorageObject { + key: object.key, + size: object.size, + e_tag: object.e_tag, + }) + .collect::>(); + if tenant_objects.iter().any(|object| object.e_tag.is_none()) { + return Err(permanent( + "object store omitted ETag for a tenant binding; conditional deletion is unavailable", + )); + } + let mut tenant_objects = tenant_objects; + tenant_objects.sort(); + let mut unsupported_version_keys = Vec::new(); - for key in bucket.tenant_keys() { + for key in &tenant_keys { if matches!( - services.media.inspect_current_version(&key).await?, + services.media.inspect_current_version(key).await?, CurrentObjectVersion::Present { - version_id: Some(_) + version_id: Some(_), + .. } ) { - unsupported_version_keys.push(key); + unsupported_version_keys.push(key.clone()); } } let storage = StorageManifest { - version: 1, - tenant_keys: bucket.tenant_keys(), + version: 2, + tenant_keys, + tenant_objects, git_pointer_keys: bucket.git_pointer_keys, media_sidecar_keys: bucket.media_sidecar_keys, media_upload_keys: bucket.media_upload_keys, @@ -552,6 +598,17 @@ async fn build_storage_manifest( Ok(storage) } +async fn build_storage_manifest( + services: &Services, + request: &DeletionRequest, +) -> Result { + let objects = services + .media + .list_all_for_deletion(storage_object_cap()) + .await?; + build_storage_manifest_from_objects(services, request, objects).await +} + async fn run_loop( services: Services, mode: LoopMode, @@ -641,6 +698,28 @@ async fn stop_claim_executor( Ok(()) } +async fn record_stage_failure( + services: &Services, + token: &LeaseToken, + stage: DeletionStage, + error: &anyhow::Error, +) -> Result { + let message = format!("{error:#}"); + let result = if is_permanent_error(error) { + services.store.block(token, stage, "stage", &message).await + } else { + services + .store + .record_retry(token, stage, "stage", &message, RETRY_DELAY) + .await + }; + match result { + Ok(()) => Ok(true), + Err(error) if buzz_db::deletion::is_stale_deletion_lease(&error) => Ok(false), + Err(error) => Err(error.into()), + } +} + async fn execute_claim( services: &Services, mode: LoopMode, @@ -677,17 +756,11 @@ async fn execute_claim( { return Ok(run_output(request)); } - let message = format!("{error:#}"); - if is_permanent_error(&error) { - services - .store - .block(&token, claim.request.stage, "stage", &message) - .await?; - } else { - services - .store - .record_retry(&token, claim.request.stage, "stage", &message, RETRY_DELAY) - .await?; + if !record_stage_failure(services, &token, claim.request.stage, &error).await? { + // Ownership expired between the stage failure and durable + // error recording. A successor now owns retry/block policy. + let request = services.store.get(token.request_id).await?; + return Ok(run_output(request)); } let request = services.store.get(token.request_id).await?; return Ok(run_output(request)); @@ -862,6 +935,7 @@ async fn execute_stage( "approved storage taxonomy drifted before fencing", )); } + services.store.begin_quiescing(&token).await?; match services.store.fence(&token).await { Ok(_) => {} Err(buzz_db::DbError::ServingWritesNotDrained { @@ -921,62 +995,98 @@ async fn execute_stage( .context("request has no post-fence destructive storage manifest")?, )?; buzz_db::deletion::validate_storage_manifest(&storage)?; - let live_storage = build_storage_manifest(services, request).await?; - if live_storage != storage { - return Err(permanent( - "post-fence storage inventory drifted before binding removal", - )); - } - for key in &storage.tenant_keys { - match services.media.inspect_current_version(key).await? { - CurrentObjectVersion::Present { version_id: None } => {} + let completed = services.store.completed_storage_object_keys(&token).await?; + let mut processed = 0usize; + for object in storage + .tenant_objects + .iter() + .filter(|object| !completed.contains(&object.key)) + .take(storage_delete_batch_size()) + { + let already_missing = match services + .media + .inspect_current_object(&object.key) + .await? + { + CurrentObjectVersion::Missing => { + if !missing_binding_is_resumable(completed.len()) { + return Err(permanent(format!( + "fenced object binding disappeared before any deletion progress: {}", + object.key + ))); + } + true + } CurrentObjectVersion::Present { version_id: Some(_), + .. } => { return Err(permanent(format!( - "unsupported object version after fence: {key}" + "unsupported object version after fence: {}", + object.key ))) } - CurrentObjectVersion::Missing => { - return Err(permanent(format!( - "fenced object binding disappeared before deletion: {key}" - ))) - } - } - } - for key in &storage.tenant_keys { - run_guarded_external_step( - services, - &token, - DeletionStage::Drained, - heartbeat_lost, - || async { - services.media.delete(key).await?; - Ok(()) - }, - ) - .await?; - run_guarded_external_step( - services, - &token, - DeletionStage::Drained, - heartbeat_lost, - || async { - if services.media.head(key).await? { - return Err(transient(format!( - "object binding still exists after delete: {key}" + CurrentObjectVersion::Present { + version_id: None, + size, + e_tag, + .. + } => { + if !storage_object_matches(object, size, e_tag.as_deref()) { + return Err(permanent(format!( + "fenced object binding changed before deletion: {}", + object.key ))); } - Ok(()) - }, - ) - .await?; + run_guarded_external_step( + services, + &token, + DeletionStage::Drained, + heartbeat_lost, + || async { + services.media.delete(&object.key).await?; + Ok(()) + }, + ) + .await?; + run_guarded_external_step( + services, + &token, + DeletionStage::Drained, + heartbeat_lost, + || async { + if services.media.head(&object.key).await? { + return Err(transient(format!( + "object binding still exists after delete: {}", + object.key + ))); + } + Ok(()) + }, + ) + .await?; + false + } + }; + services + .store + .checkpoint_storage_object_removed(&token, &object.key, already_missing) + .await?; + processed += 1; + } + + let completed_count = completed.len().saturating_add(processed); + if completed_count < storage.tenant_objects.len() { + return Err(transient(format!( + "storage deletion batch complete: {completed_count}/{}", + storage.tenant_objects.len() + ))); } services .store .mark_bindings_removed( &token, - serde_json::json!({"deleted_keys": storage.tenant_keys.len()}), + serde_json::json!({"deleted_keys": storage.tenant_objects.len()}), ) .await?; } @@ -1046,21 +1156,29 @@ fn token_with_current_fence(token: &LeaseToken, request: &DeletionRequest) -> Le } } -async fn verify_storage_absence(services: &Services, request: &DeletionRequest) -> Result<()> { - let storage: StorageManifest = serde_json::from_value( - request - .destructive_storage_manifest - .clone() - .context("request has no post-fence destructive storage manifest")?, - )?; - for key in &storage.tenant_keys { - if services.media.head(key).await? { - return Err(transient(format!( - "logical verification found object binding: {key}" - ))); - } +async fn verify_storage_absence_from_objects( + services: &Services, + request: &DeletionRequest, + objects: Vec, +) -> Result<()> { + let live = build_storage_manifest_from_objects(services, request, objects).await?; + if live.tenant_keys.is_empty() { + Ok(()) + } else { + Err(transient(format!( + "logical verification found {} live target object binding(s); first={}", + live.tenant_keys.len(), + live.tenant_keys.first().map_or("", String::as_str) + ))) } - Ok(()) +} + +async fn verify_storage_absence(services: &Services, request: &DeletionRequest) -> Result<()> { + let objects = services + .media + .list_all_for_deletion(storage_object_cap()) + .await?; + verify_storage_absence_from_objects(services, request, objects).await } async fn publish_disconnect_community( @@ -1164,6 +1282,15 @@ fn storage_object_cap() -> u64 { .unwrap_or(DEFAULT_STORAGE_OBJECT_CAP) } +fn storage_delete_batch_size() -> usize { + std::env::var("BUZZ_DELETION_STORAGE_DELETE_BATCH_SIZE") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .map(|value| value.min(10_000)) + .unwrap_or(STORAGE_DELETE_BATCH_SIZE) +} + fn default_executor_id() -> String { let hostname = std::env::var("HOSTNAME").unwrap_or_else(|_| "buzz-admin".to_string()); format!("{hostname}:{}", std::process::id()) @@ -1278,8 +1405,9 @@ mod tests { .await .expect("inventory schema"), storage: StorageManifest { - version: 1, + version: 2, tenant_keys: Vec::new(), + tenant_objects: Vec::new(), git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), @@ -1329,6 +1457,51 @@ mod tests { Some((services, claim)) } + #[test] + fn large_storage_work_is_bounded_per_attempt() { + assert_eq!(STORAGE_DELETE_BATCH_SIZE, 100); + let keys = (0..1_000_000).take(STORAGE_DELETE_BATCH_SIZE).count(); + assert_eq!(keys, STORAGE_DELETE_BATCH_SIZE); + } + + #[test] + fn storage_delete_batches_resume_from_completed_checkpoints() { + let objects = (0..250) + .map(|index| StorageObject { + key: format!("key-{index:03}"), + size: 1, + e_tag: Some(format!("etag-{index}")), + }) + .collect::>(); + let completed = objects[..100] + .iter() + .map(|object| object.key.clone()) + .collect::>(); + let resumed = objects + .iter() + .filter(|object| !completed.contains(&object.key)) + .take(STORAGE_DELETE_BATCH_SIZE) + .map(|object| object.key.clone()) + .collect::>(); + assert_eq!(resumed.len(), 100); + assert_eq!(resumed.first().map(String::as_str), Some("key-100")); + assert_eq!(resumed.last().map(String::as_str), Some("key-199")); + assert!(missing_binding_is_resumable(completed.len())); + assert!(!missing_binding_is_resumable(0)); + } + + #[test] + fn object_identity_rejects_replacement_but_accepts_exact_match() { + let expected = StorageObject { + key: "tenant/key".to_string(), + size: 42, + e_tag: Some("etag-a".to_string()), + }; + assert!(storage_object_matches(&expected, 42, Some("etag-a"))); + assert!(!storage_object_matches(&expected, 42, Some("etag-b"))); + assert!(!storage_object_matches(&expected, 42, None)); + } + #[test] fn permanent_failures_are_typed_not_string_classified() { let permanent_error = permanent("catalog drift"); @@ -1383,6 +1556,57 @@ mod tests { assert!(scan_proves_absence(&[(9, Vec::new()), (0, Vec::new())])); } + #[tokio::test] + async fn final_storage_verification_rejects_late_target_binding() { + let Some((services, claim)) = claimed_test_deletion("deletion-late-binding").await else { + return; + }; + let community = claim.request.community_id; + let late_key = format!("_meta/{community}/{}.json", "a".repeat(64)); + let error = verify_storage_absence_from_objects( + &services, + &claim.request, + vec![buzz_media::DeletionObject { + key: late_key.clone(), + size: 7, + last_modified: "2026-08-02T00:00:00Z".to_string(), + e_tag: Some("etag".to_string()), + }], + ) + .await + .expect_err("late target binding must fail verification"); + assert!(format!("{error:#}").contains(&late_key)); + } + + #[tokio::test] + async fn stale_lease_during_failure_recording_is_lost_ownership() { + let Some((services, claim)) = claimed_test_deletion("deletion-stale-record").await else { + return; + }; + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .expect("test database URL"); + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect stale-record test DB"); + sqlx::query( + "UPDATE community_deletion_requests SET lease_until = now() - interval '1 second' WHERE id = $1", + ) + .bind(claim.request.id) + .execute(&pool) + .await + .expect("expire claim"); + let recorded = record_stage_failure( + &services, + &claim.lease, + claim.request.stage, + &transient("test failure"), + ) + .await + .expect("stale ownership is not fatal"); + assert!(!recorded); + } + #[tokio::test] async fn serving_guard_cancels_protected_operation_when_heartbeat_is_lost() { let database_url = match std::env::var("BUZZ_TEST_DATABASE_URL") diff --git a/crates/buzz-media/src/error.rs b/crates/buzz-media/src/error.rs index c3d180402f..14ce4afe1e 100644 --- a/crates/buzz-media/src/error.rs +++ b/crates/buzz-media/src/error.rs @@ -54,6 +54,10 @@ pub enum MediaError { InsufficientScope, #[error("relay membership required")] RelayMembershipRequired, + #[error("community writes are fenced")] + CommunityWriteFenced, + #[error("media service temporarily unavailable")] + ServiceUnavailable, #[error("token revoked")] TokenRevoked, #[error("pubkey mismatch")] @@ -138,7 +142,10 @@ impl IntoResponse for MediaError { ) } Self::InsufficientScope => (StatusCode::FORBIDDEN, self.to_string()), - Self::RelayMembershipRequired => (StatusCode::FORBIDDEN, self.to_string()), + Self::RelayMembershipRequired | Self::CommunityWriteFenced => { + (StatusCode::FORBIDDEN, self.to_string()) + } + Self::ServiceUnavailable => (StatusCode::SERVICE_UNAVAILABLE, self.to_string()), Self::UploadRateLimitExceeded | Self::UploadConcurrencyLimitReached => { (StatusCode::TOO_MANY_REQUESTS, self.to_string()) } @@ -164,6 +171,21 @@ impl IntoResponse for MediaError { mod tests { use super::*; + #[test] + fn serving_backend_failures_map_to_5xx_but_fences_remain_403() { + for error in [ + MediaError::ServiceUnavailable, + MediaError::Internal, + MediaError::StorageError("backend".to_string()), + ] { + assert!(error.into_response().status().is_server_error()); + } + assert_eq!( + MediaError::CommunityWriteFenced.into_response().status(), + StatusCode::FORBIDDEN + ); + } + #[test] fn unsupported_media_maps_to_415() { for error in [ diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index 67c117e8df..f2a0ca7158 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -202,6 +202,7 @@ impl MediaStorage { key: object.key, size: object.size, last_modified: object.last_modified, + e_tag: object.e_tag, })); if !result.is_truncated { break; @@ -216,24 +217,35 @@ impl MediaStorage { Ok(objects) } - /// Return the current object version id, if the backend reports one. - /// - /// V1 refuses to delete versioned tenant bindings because rust-s3 cannot - /// exhaustively enumerate non-current versions; a plain delete marker would - /// not prove logical absence from storage. - pub async fn inspect_current_version( + /// Return the current object identity, including whether it is versioned. + pub async fn inspect_current_object( &self, key: &str, ) -> Result { match self.bucket.head_object(key).await { Ok((result, _)) => Ok(CurrentObjectVersion::Present { version_id: result.version_id, + size: result.content_length.unwrap_or_default().max(0) as u64, + last_modified: result.last_modified, + e_tag: result.e_tag, }), Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(CurrentObjectVersion::Missing), Err(error) => Err(MediaError::StorageError(error.to_string())), } } + /// Return the current object version id, if the backend reports one. + /// + /// V1 refuses to delete versioned tenant bindings because rust-s3 cannot + /// exhaustively enumerate non-current versions; a plain delete marker would + /// not prove logical absence from storage. + pub async fn inspect_current_version( + &self, + key: &str, + ) -> Result { + self.inspect_current_object(key).await + } + /// Delete tenant binding keys one at a time and fail on any backend error. pub async fn delete_bindings(&self, keys: &[String]) -> Result<(), MediaError> { for key in keys { @@ -348,6 +360,12 @@ pub enum CurrentObjectVersion { Present { /// S3 version id. `Some` is unsupported for V1 deletion. version_id: Option, + /// Current object size. + size: u64, + /// Current last-modified timestamp, when supplied. + last_modified: Option, + /// Current entity tag, when supplied. + e_tag: Option, }, } @@ -360,6 +378,8 @@ pub struct DeletionObject { pub size: u64, /// Backend-reported last modification timestamp. pub last_modified: String, + /// Backend entity tag, when supplied. + pub e_tag: Option, } #[cfg(test)] diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index 144fbf846b..b01b4561b5 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -2339,8 +2339,9 @@ mod track_c_tests { .await .expect("schema inventory"), storage: StorageManifest { - version: 1, + version: 2, tenant_keys: Vec::new(), + tenant_objects: Vec::new(), git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), @@ -2454,6 +2455,12 @@ mod track_c_tests { tokio::spawn(async move { finalize_push_inner(&finalize_state, ctx, &hooks).await }); gate.reached.notified().await; + state + .db + .deletion_store() + .begin_quiescing(&claim.lease) + .await + .expect("quiesce after CAS"); let error = state .db .deletion_store() @@ -2464,12 +2471,12 @@ mod track_c_tests { error, buzz_db::DbError::ServingWritesNotDrained { .. } )); - assert!(state + assert!(!state .db .deletion_store() .is_serving_active(community) .await - .expect("community remains active")); + .expect("quiescing rejects new serving work")); gate.resume.notify_one(); let response = finalize.await.expect("finalize task"); diff --git a/crates/buzz-relay/src/api/media.rs b/crates/buzz-relay/src/api/media.rs index 3f84405827..cfc7750ed4 100644 --- a/crates/buzz-relay/src/api/media.rs +++ b/crates/buzz-relay/src/api/media.rs @@ -283,6 +283,19 @@ async fn upload_attribution( }) } +fn serving_write_error(error: anyhow::Error) -> MediaError { + if buzz_deletion::ServingWriteGuard::acquisition_is_fenced(&error) { + MediaError::CommunityWriteFenced + } else { + MediaError::ServiceUnavailable + } +} + +fn serving_lease_lost(error: anyhow::Error) -> MediaError { + tracing::warn!(%error, "media serving-write lease lost"); + MediaError::ServiceUnavailable +} + /// PUT `/upload` or the temporary media-only `/media/upload` alias. /// /// Auth is validated via the [`AuthenticatedUpload`] extractor BEFORE the body @@ -313,7 +326,7 @@ pub async fn upload_blob( let serving_write = buzz_deletion::acquire_serving_write(&state.db, auth.tenant.community(), "media_upload") .await - .map_err(|_| MediaError::RelayMembershipRequired)?; + .map_err(serving_write_error)?; if auth.route_mode == UploadRouteMode::LegacyMedia { metrics::counter!("buzz_media_legacy_upload_route_total").increment(1); @@ -340,10 +353,7 @@ pub async fn upload_blob( } let replay = futures_util::stream::iter(replay_chunks.into_iter().map(Ok)).chain(source); - serving_write - .verify() - .await - .map_err(|_| MediaError::RelayMembershipRequired)?; + serving_write.verify().await.map_err(serving_lease_lost)?; let mut descriptor = serving_write .protect(async { @@ -415,7 +425,7 @@ pub async fn upload_blob( .await .map_err(|error| { if buzz_deletion::ServingWriteGuard::is_lease_lost(&error) { - MediaError::RelayMembershipRequired + serving_lease_lost(error) } else { match error.downcast::() { Ok(error) => error, @@ -466,10 +476,7 @@ pub async fn upload_blob( } } - serving_write - .finish() - .await - .map_err(|_| MediaError::RelayMembershipRequired)?; + serving_write.finish().await.map_err(serving_lease_lost)?; Ok(Json(descriptor)) } @@ -951,6 +958,20 @@ mod tests { const VALID_HASH: &str = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + #[test] + fn serving_write_error_taxonomy_separates_fence_from_backend_failure() { + let fenced = anyhow::Error::from(buzz_db::DbError::AccessDenied("fenced".to_string())); + assert!(matches!( + serving_write_error(fenced), + MediaError::CommunityWriteFenced + )); + let backend = anyhow::Error::from(buzz_db::DbError::Sqlx(sqlx::Error::PoolTimedOut)); + assert!(matches!( + serving_write_error(backend), + MediaError::ServiceUnavailable + )); + } + #[test] fn upload_routes_distinguish_standard_and_legacy_modes() { assert_eq!( diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index 3a63727986..05865ece64 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -727,25 +727,31 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc EphemeralTerminal::Accepted, + Ok(Err(message)) => { + reject("invalid"); + EphemeralTerminal::Rejected(message) + } + Err(error) => { + reject("restricted"); + EphemeralTerminal::LeaseLost(format!( + "restricted: community write lease lost: {error}" + )) + } + }; + if matches!(terminal, EphemeralTerminal::Accepted) { + if let Err(error) = serving_write.finish().await { + tracing::warn!(%error, event_id = %event_id_hex, "failed to release ephemeral-event serving lease"); + } } + conn.send(terminal.response(&event_id_hex)); return; } @@ -789,37 +795,40 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc String { + match self { + Self::Accepted => RelayMessage::ok(event_id, true, ""), + Self::Rejected(message) | Self::LeaseLost(message) => { + RelayMessage::ok(event_id, false, message) + } + } + } +} + /// Handle ephemeral events (kind 20000–29999) — WS-only, never stored. async fn handle_ephemeral_event( event: Event, conn_id: uuid::Uuid, - event_id_hex: &str, pubkey_bytes: Vec, auth_pubkey: nostr::PublicKey, conn: Arc, state: Arc, -) { +) -> Result<(), String> { let event_clone = event.clone(); + let event_id = event.id.to_hex(); let verify_result = tokio::task::spawn_blocking(move || verify_event(&event_clone)).await; match verify_result { Ok(Ok(())) => {} - Ok(Err(e)) => { - conn.send(RelayMessage::ok( - event_id_hex, - false, - &format!("invalid: {e}"), - )); - return; - } - Err(_) => { - conn.send(RelayMessage::ok( - event_id_hex, - false, - "error: internal error", - )); - return; - } + Ok(Err(e)) => return Err(format!("invalid: {e}")), + Err(_) => return Err("error: internal error".to_string()), } // Special handling for presence events (kind:20001). @@ -860,18 +869,8 @@ async fn handle_ephemeral_event( // Check channel membership before publishing other ephemeral events. if let Some(ch_id) = super::ingest::extract_channel_id(&event) { - if let Err(msg) = super::ingest::check_channel_membership( - &conn.tenant, - &state, - ch_id, - &pubkey_bytes, - None, - ) - .await - { - conn.send(RelayMessage::ok(event_id_hex, false, &msg)); - return; - } + super::ingest::check_channel_membership(&conn.tenant, &state, ch_id, &pubkey_bytes, None) + .await?; // Mark as local before Redis publish to prevent double-delivery when // the event comes back through the Redis subscriber loop. @@ -885,7 +884,7 @@ async fn handle_ephemeral_event( state .local_event_ids .invalidate(&(conn.tenant.community(), event.id.to_bytes())); - warn!(conn_id = %conn_id, event_id = %event_id_hex, "Ephemeral publish failed: {e}"); + warn!(conn_id = %conn_id, event_id = %event_id, "Ephemeral publish failed: {e}"); } // Direct fan-out to local WS subscribers, through the guarded send path @@ -913,7 +912,7 @@ async fn handle_ephemeral_event( state .local_event_ids .invalidate(&(conn.tenant.community(), event.id.to_bytes())); - warn!(conn_id = %conn_id, event_id = %event_id_hex, "Ephemeral global publish failed: {e}"); + warn!(conn_id = %conn_id, event_id = %event_id, "Ephemeral global publish failed: {e}"); } // Direct fan-out to local WS subscribers through the guarded send path. @@ -924,7 +923,7 @@ async fn handle_ephemeral_event( fan_out_event_to_local_subscribers(&state, conn.tenant.community(), &stored_event).await; } - conn.send(RelayMessage::ok(event_id_hex, true, "")); + Ok(()) } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1204,6 +1203,25 @@ mod tests { use tokio_util::sync::CancellationToken; use uuid::Uuid; + #[test] + fn ephemeral_terminal_builds_one_unambiguous_ok() { + let event_id = "abc"; + assert_eq!( + serde_json::from_str::( + &super::EphemeralTerminal::Accepted.response(event_id) + ) + .unwrap(), + serde_json::json!(["OK", event_id, true, ""]) + ); + assert_eq!( + serde_json::from_str::( + &super::EphemeralTerminal::LeaseLost("restricted".to_string()).response(event_id) + ) + .unwrap(), + serde_json::json!(["OK", event_id, false, "restricted"]) + ); + } + #[test] fn fanout_event_frame_matches_legacy_format_byte_for_byte() { let sub_id = "sub-id"; diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index e545214917..3df2f4b42b 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -3171,6 +3171,7 @@ mod tests { .unwrap_or_else(|_| "postgres://buzz:buzz_dev@localhost:5432/buzz".to_string()); // sadscan:disable np.postgres.1 let pool = sqlx::PgPool::connect(&url).await.expect("connect test DB"); let db = buzz_db::Db::from_pool(pool); + db.migrate().await.expect("migrate test DB"); let store = buzz_deletion::store(&db); let host = format!("lane3-fence-{}.example", Uuid::new_v4().simple()); @@ -3199,8 +3200,9 @@ mod tests { .await .expect("schema inventory"), storage: StorageManifest { - version: 1, + version: 2, tenant_keys: Vec::new(), + tenant_objects: Vec::new(), git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), @@ -3222,6 +3224,7 @@ mod tests { .await .expect("claim") .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); store.fence(&claim.lease).await.expect("fence"); match map_serving_fence_state(store.is_serving_active(community).await) { diff --git a/crates/buzz-relay/src/main.rs b/crates/buzz-relay/src/main.rs index cc6fe74718..3584e1849d 100644 --- a/crates/buzz-relay/src/main.rs +++ b/crates/buzz-relay/src/main.rs @@ -201,11 +201,11 @@ async fn main() -> anyhow::Result<()> { error!("Failed to ensure partitions: {e}"); } - db.validate_deletion_catalog().await.map_err(|e| { - error!("Community deletion catalog validation failed: {e}"); - anyhow::anyhow!("Community deletion catalog is unsafe: {e}") + db.validate_deletion_serving_catalog().await.map_err(|e| { + error!("Community deletion serving-fence validation failed: {e}"); + anyhow::anyhow!("Community deletion serving fence is unsafe: {e}") })?; - info!("Community deletion catalog and write fences verified"); + info!("Community deletion serving fences verified"); // Freshness fence probe: cursor pages route to the replica only for // history the probe has verified as fully replayed. Deliberately AFTER @@ -1025,6 +1025,24 @@ async fn main() -> anyhow::Result<()> { metrics::gauge!("buzz_redis_pool_size").set(rs.size as f64); metrics::gauge!("buzz_redis_pool_max").set(rs.max_size as f64); metrics::gauge!("buzz_redis_pool_waiting").set(rs.waiting as f64); + + let deletion_store = pool_state.db.deletion_store(); + match deletion_store.reap_expired_serving_write_leases(1000).await { + Ok(reaped) => metrics::counter!("buzz_deletion_serving_leases_reaped_total") + .increment(reaped), + Err(error) => tracing::warn!(%error, "serving-lease reaper failed"), + } + match deletion_store.serving_lease_stats().await { + Ok(stats) => { + metrics::gauge!("buzz_deletion_serving_leases_active") + .set(stats.active as f64); + metrics::gauge!("buzz_deletion_serving_leases_expired") + .set(stats.expired as f64); + metrics::gauge!("buzz_deletion_serving_leases_dead_tuples") + .set(stats.dead_tuples as f64); + } + Err(error) => tracing::warn!(%error, "serving-lease metrics failed"), + } } }); } diff --git a/crates/buzz-relay/src/router.rs b/crates/buzz-relay/src/router.rs index f0daa5a73d..82ad9938a2 100644 --- a/crates/buzz-relay/src/router.rs +++ b/crates/buzz-relay/src/router.rs @@ -379,7 +379,7 @@ async fn readiness_handler(State(state): State>) -> impl IntoRespo let (pg_ok, redis_ok, deletion_catalog_ok) = tokio::join!( state.db.ping(), async { state.redis_pool.get().await.is_ok() }, - async { state.db.validate_deletion_catalog().await.is_ok() }, + async { state.db.validate_deletion_serving_catalog().await.is_ok() }, ); (pg_ok, redis_ok, deletion_catalog_ok) }; diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index c04bb47ca2..2ecc5d4440 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -176,6 +176,36 @@ listener exists only for pod probes, and SIGTERM marks readiness draining, finishes/releases the current durable unit, then exits inside the configured 90-second window. Do not route public traffic to this pod. +### Required deletion rollout order + +Migration 0027 installs database triggers, but replicas built before this +release do not acquire the external-effect leases that cover S3, Redis, push, +and workflow work. **Do not approve or run a deletion until every serving +replica is this release or newer.** Use this order: + +1. Disable/scale the deletion worker to zero and leave requests unapproved. +2. Apply migration 0027 during a low-traffic window. It sets a 5-second + `lock_timeout`; if any trigger's `SHARE ROW EXCLUSIVE` lock cannot be acquired, + the transaction fails with no partial catalog and should be retried after + clearing/finishing long transactions. +3. Roll every relay replica and verify all old ReplicaSets are at zero plus + `/_readiness` is healthy. Serving readiness accepts migration 27 and later; + destructive claims intentionally require exact migration 27/catalog equality. +4. Verify `buzz_deletion_serving_leases_active` responds to test traffic and + `buzz_deletion_serving_leases_expired` stays near zero. Alert on growing + `buzz_deletion_serving_leases_dead_tuples` and inspect autovacuum if it grows. +5. Enable the worker and only then approve/run requests. + +PostgreSQL trigger attachment and lease-table maintenance add write cost. +Track application p99 for events/media/Git/workflows alongside the lease gauges; +investigate rising dead tuples or expired rows before raising worker scale. + +Object removal is checkpointed per key and each attempt deletes at most +`deletionWorker.storageDeleteBatchSize` (default 100). After recorded deletion +progress, missing keys resume as already-complete; a missing key before any +checkpoint still blocks as unexplained drift. Changed/versioned keys fail closed. +Final verification performs a fresh complete target inventory before advancing. + ## Device pairing relay The chart can run Buzz's stateless pairing WebSocket relay as an independent @@ -230,7 +260,7 @@ default so long-lived WebSocket connections have time to drain. Schema migrations are embedded in the relay binary via `sqlx::migrate!` and run at startup, gated by `BUZZ_AUTO_MIGRATE` (default `true`). Multiple replicas race-safely behind a Postgres advisory lock. `helm upgrade` is the entire upgrade procedure. -If you prefer decoupling migrations from serving, set `migrate.autoMigrate=false`. **In that mode the chart does not run migrations for you** — you own running `buzz-admin migrate` (separate Pod / one-shot Job) against the database before every `helm install` / `helm upgrade`. Readiness probes only verify DB connectivity, not schema freshness, so a pod will appear healthy against an unmigrated schema and fail under load. A pre-upgrade Helm Job for this is on the chart roadmap; the values knob `migrate.preUpgradeJob.enabled` is reserved. +If you prefer decoupling migrations from serving, set `migrate.autoMigrate=false`. **In that mode the chart does not run migrations for you** — you own running `buzz-admin migrate` (separate Pod / one-shot Job) against the database before every `helm install` / `helm upgrade`. This release's startup/readiness check requires migration 0027's serving-fence objects (while accepting later additive migrations), so 0027 must finish before deploying these relay pods. Other feature-specific schema freshness remains operator-owned. A pre-upgrade Helm Job for this is on the chart roadmap; the values knob `migrate.preUpgradeJob.enabled` is reserved. ## Backups diff --git a/deploy/charts/buzz/templates/deletion-worker-deployment.yaml b/deploy/charts/buzz/templates/deletion-worker-deployment.yaml index 42f02fb6d9..826846002d 100644 --- a/deploy/charts/buzz/templates/deletion-worker-deployment.yaml +++ b/deploy/charts/buzz/templates/deletion-worker-deployment.yaml @@ -48,6 +48,7 @@ spec: - { name: RELAY_URL, value: {{ .Values.relayUrl | quote }} } - { name: BUZZ_DELETION_HEALTH_ADDR, value: {{ printf "0.0.0.0:%v" .Values.deletionWorker.healthPort | quote }} } - { name: BUZZ_DELETION_STORAGE_MAX_OBJECTS, value: {{ .Values.deletionWorker.storageMaxObjects | quote }} } + - { name: BUZZ_DELETION_STORAGE_DELETE_BATCH_SIZE, value: {{ .Values.deletionWorker.storageDeleteBatchSize | quote }} } {{- $s3Endpoint := include "buzz.s3Endpoint" . }} {{- if $s3Endpoint }} - { name: BUZZ_S3_ENDPOINT, value: {{ $s3Endpoint | quote }} } diff --git a/deploy/charts/buzz/tests/deletion_worker_test.yaml b/deploy/charts/buzz/tests/deletion_worker_test.yaml index 53730b0a94..a664cb2ba6 100644 --- a/deploy/charts/buzz/tests/deletion_worker_test.yaml +++ b/deploy/charts/buzz/tests/deletion_worker_test.yaml @@ -43,7 +43,10 @@ tests: - equal: path: spec.template.spec.containers[0].readinessProbe.httpGet.path value: /_readiness - - notExists: - path: spec.template.spec.containers[0].env[8].valueFrom.secretKeyRef.optional + - equal: + path: spec.template.spec.containers[0].env[3].value + value: "100" - notExists: path: spec.template.spec.containers[0].env[9].valueFrom.secretKeyRef.optional + - notExists: + path: spec.template.spec.containers[0].env[10].valueFrom.secretKeyRef.optional diff --git a/deploy/charts/buzz/values.schema.json b/deploy/charts/buzz/values.schema.json index e1e362a531..2777fea512 100644 --- a/deploy/charts/buzz/values.schema.json +++ b/deploy/charts/buzz/values.schema.json @@ -270,6 +270,24 @@ "labels": { "type": "object" } } }, + "deletionWorker": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { "type": "boolean" }, + "replicaCount": { "type": "integer", "minimum": 1 }, + "serviceAccountName": { "type": "string" }, + "automountServiceAccountToken": { "type": "boolean" }, + "existingSecret": { "type": "string" }, + "healthPort": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 1 }, + "storageMaxObjects": { "type": "integer", "minimum": 1 }, + "storageDeleteBatchSize": { "type": "integer", "minimum": 1, "maximum": 10000 }, + "podAnnotations": { "type": "object" }, + "extraEnv": { "type": "array", "items": { "type": "object" } }, + "resources": { "type": "object" } + } + }, "pairingRelay": { "type": "object", "additionalProperties": false, diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index 72ddb55072..cced4ecca9 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -411,6 +411,8 @@ deletionWorker: healthPort: 8080 terminationGracePeriodSeconds: 90 storageMaxObjects: 1000000 + # Durable per-key checkpoints bound one worker attempt to this many deletes. + storageDeleteBatchSize: 100 podAnnotations: {} extraEnv: [] resources: diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index b31690f533..72e99d1a2f 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -3,9 +3,15 @@ -- The community row is never removed: it becomes the permanent name tombstone. -- All destructive progress is lease/fence/checkpoint guarded and every existing -- community-scoped table receives the same database-enforced write fence. +-- This migration intentionally remains one atomic catalog change so a failed +-- deployment cannot expose only a subset of the universal fences. CREATE +-- TRIGGER takes SHARE ROW EXCLUSIVE on each target; fail quickly rather than +-- queueing behind long transactions. See the chart deletion rollout runbook. +SET LOCAL lock_timeout = '5s'; + ALTER TABLE communities ADD COLUMN deletion_state TEXT NOT NULL DEFAULT 'active' - CHECK (deletion_state IN ('active', 'fenced', 'tombstone')), + CHECK (deletion_state IN ('active', 'quiescing', 'fenced', 'tombstone')), ADD COLUMN deletion_fence_generation BIGINT NOT NULL DEFAULT 0 CHECK (deletion_fence_generation >= 0), ADD COLUMN deleted_at TIMESTAMPTZ; diff --git a/schema/schema.sql b/schema/schema.sql index 6e9901c904..902033b1a4 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -59,7 +59,7 @@ CREATE TABLE communities ( icon TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), archived_at TIMESTAMPTZ, - deletion_state TEXT NOT NULL DEFAULT 'active' CHECK (deletion_state IN ('active', 'fenced', 'tombstone')), + deletion_state TEXT NOT NULL DEFAULT 'active' CHECK (deletion_state IN ('active', 'quiescing', 'fenced', 'tombstone')), deletion_fence_generation BIGINT NOT NULL DEFAULT 0 CHECK (deletion_fence_generation >= 0), deleted_at TIMESTAMPTZ, CONSTRAINT chk_communities_id_not_nil CHECK (id <> '00000000-0000-0000-0000-000000000000'::uuid) From 3e5e4244654b5b8e76d29a08f2b92663c79d1f53 Mon Sep 17 00:00:00 2001 From: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 11:02:02 -0700 Subject: [PATCH 09/51] fix(deletion): resume an uncheckpointed first object Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Co-authored-by: Kalvin Chau Signed-off-by: Kalvin Chau --- crates/buzz-deletion/src/lib.rs | 173 ++++++++++++++++++++------------ 1 file changed, 110 insertions(+), 63 deletions(-) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index df10e60c6b..f0f6f35969 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -517,10 +517,6 @@ fn storage_taxonomy_matches(approved: &StorageManifest, live: &StorageManifest) && approved.unsupported_version_keys == live.unsupported_version_keys } -fn missing_binding_is_resumable(completed_count: usize) -> bool { - completed_count > 0 -} - fn storage_object_matches( expected: &StorageObject, observed_size: u64, @@ -529,6 +525,40 @@ fn storage_object_matches( expected.size == observed_size && observed_e_tag == expected.e_tag.as_deref() } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StorageObjectAction { + AlreadyMissing, + Delete, +} + +fn storage_object_action( + expected: &StorageObject, + current: &CurrentObjectVersion, +) -> Result { + match current { + CurrentObjectVersion::Missing => Ok(StorageObjectAction::AlreadyMissing), + CurrentObjectVersion::Present { + version_id: Some(_), + .. + } => Err(permanent(format!( + "unsupported object version after fence: {}", + expected.key + ))), + CurrentObjectVersion::Present { + version_id: None, + size, + e_tag, + .. + } if storage_object_matches(expected, *size, e_tag.as_deref()) => { + Ok(StorageObjectAction::Delete) + } + CurrentObjectVersion::Present { .. } => Err(permanent(format!( + "fenced object binding changed before deletion: {}", + expected.key + ))), + } +} + async fn build_inventory( services: &Services, request: &DeletionRequest, @@ -1003,41 +1033,10 @@ async fn execute_stage( .filter(|object| !completed.contains(&object.key)) .take(storage_delete_batch_size()) { - let already_missing = match services - .media - .inspect_current_object(&object.key) - .await? - { - CurrentObjectVersion::Missing => { - if !missing_binding_is_resumable(completed.len()) { - return Err(permanent(format!( - "fenced object binding disappeared before any deletion progress: {}", - object.key - ))); - } - true - } - CurrentObjectVersion::Present { - version_id: Some(_), - .. - } => { - return Err(permanent(format!( - "unsupported object version after fence: {}", - object.key - ))) - } - CurrentObjectVersion::Present { - version_id: None, - size, - e_tag, - .. - } => { - if !storage_object_matches(object, size, e_tag.as_deref()) { - return Err(permanent(format!( - "fenced object binding changed before deletion: {}", - object.key - ))); - } + let current = services.media.inspect_current_object(&object.key).await?; + let already_missing = match storage_object_action(object, ¤t)? { + StorageObjectAction::AlreadyMissing => true, + StorageObjectAction::Delete => { run_guarded_external_step( services, &token, @@ -1464,30 +1463,78 @@ mod tests { assert_eq!(keys, STORAGE_DELETE_BATCH_SIZE); } - #[test] - fn storage_delete_batches_resume_from_completed_checkpoints() { - let objects = (0..250) - .map(|index| StorageObject { - key: format!("key-{index:03}"), - size: 1, - e_tag: Some(format!("etag-{index}")), - }) - .collect::>(); - let completed = objects[..100] - .iter() - .map(|object| object.key.clone()) - .collect::>(); - let resumed = objects - .iter() - .filter(|object| !completed.contains(&object.key)) - .take(STORAGE_DELETE_BATCH_SIZE) - .map(|object| object.key.clone()) - .collect::>(); - assert_eq!(resumed.len(), 100); - assert_eq!(resumed.first().map(String::as_str), Some("key-100")); - assert_eq!(resumed.last().map(String::as_str), Some("key-199")); - assert!(missing_binding_is_resumable(completed.len())); - assert!(!missing_binding_is_resumable(0)); + #[tokio::test] + async fn missing_first_object_without_checkpoint_is_checkpointed_and_advanced() { + let Some((services, claim)) = claimed_test_deletion("deletion-first-missing").await else { + return; + }; + let object = StorageObject { + key: "only-key".to_string(), + size: 1, + e_tag: Some("etag".to_string()), + }; + let storage = StorageManifest { + version: 2, + tenant_keys: vec![object.key.clone()], + tenant_objects: vec![object.clone()], + git_pointer_keys: Vec::new(), + media_sidecar_keys: Vec::new(), + media_upload_keys: Vec::new(), + retained_shared_cas_keys: Vec::new(), + unknown_keys: Vec::new(), + unsupported_version_keys: Vec::new(), + }; + services + .store + .begin_quiescing(&claim.lease) + .await + .expect("quiesce"); + let generation = services.store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + services + .store + .freeze_destructive_storage_manifest(&token, &storage) + .await + .expect("freeze one-object manifest"); + services.store.mark_drained(&token).await.expect("drained"); + assert!(services + .store + .completed_storage_object_keys(&token) + .await + .expect("initial checkpoints") + .is_empty()); + + assert_eq!( + storage_object_action(&object, &CurrentObjectVersion::Missing) + .expect("missing delete result reconciles"), + StorageObjectAction::AlreadyMissing + ); + let object_key = object.key.clone(); + services + .store + .checkpoint_storage_object_removed(&token, &object_key, true) + .await + .expect("checkpoint missing object"); + assert_eq!( + services + .store + .completed_storage_object_keys(&token) + .await + .expect("completed checkpoints"), + std::collections::BTreeSet::from([object_key]) + ); + services + .store + .mark_bindings_removed(&token, serde_json::json!({"deleted_keys": 1})) + .await + .expect("advance after reconciliation"); + assert_eq!( + services.store.get(token.request_id).await.unwrap().stage, + DeletionStage::BindingsRemoved + ); } #[test] From f4734cc945e364fb2e4c324dc0d28a232373985b Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 12:37:30 -0700 Subject: [PATCH 10/51] test(deletion): exercise crash resume through drained stage Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-deletion/src/lib.rs | 113 +++++++++++++++++++++++++------- 1 file changed, 88 insertions(+), 25 deletions(-) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index f0f6f35969..1aab912d07 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -1456,6 +1456,42 @@ mod tests { Some((services, claim)) } + fn deletion_test_media_storage() -> Option> { + let endpoint = std::env::var("BUZZ_TEST_S3_ENDPOINT") + .or_else(|_| std::env::var("BUZZ_S3_ENDPOINT")) + .ok()?; + let access_key = std::env::var("BUZZ_TEST_S3_ACCESS_KEY") + .or_else(|_| std::env::var("BUZZ_S3_ACCESS_KEY")) + .ok()?; + let secret_key = std::env::var("BUZZ_TEST_S3_SECRET_KEY") + .or_else(|_| std::env::var("BUZZ_S3_SECRET_KEY")) + .ok()?; + let bucket = std::env::var("BUZZ_TEST_S3_BUCKET") + .or_else(|_| std::env::var("BUZZ_S3_BUCKET")) + .ok()?; + Some(Arc::new( + MediaStorage::new(&buzz_media::MediaConfig { + s3_endpoint: endpoint, + s3_access_key: access_key, + s3_secret_key: secret_key, + s3_bucket: bucket, + s3_region: std::env::var("BUZZ_TEST_S3_REGION") + .or_else(|_| std::env::var("BUZZ_S3_REGION")) + .unwrap_or_else(|_| "us-east-1".to_string()), + s3_addressing_style: buzz_media::S3AddressingStyle::Path, + max_image_bytes: 1, + max_gif_bytes: 1, + max_video_bytes: 1, + max_file_bytes: 1, + public_base_url: "http://localhost/media".to_string(), + upload_records_enabled: false, + upload_ip_header: None, + upload_port_header: None, + }) + .expect("construct deletion test media service"), + )) + } + #[test] fn large_storage_work_is_bounded_per_attempt() { assert_eq!(STORAGE_DELETE_BATCH_SIZE, 100); @@ -1464,21 +1500,44 @@ mod tests { } #[tokio::test] - async fn missing_first_object_without_checkpoint_is_checkpointed_and_advanced() { - let Some((services, claim)) = claimed_test_deletion("deletion-first-missing").await else { + async fn drained_stage_reconciles_first_object_deleted_before_checkpoint() { + let Some((mut services, claim)) = claimed_test_deletion("deletion-first-missing").await + else { return; }; - let object = StorageObject { - key: "only-key".to_string(), - size: 1, - e_tag: Some("etag".to_string()), + let Some(media) = deletion_test_media_storage() else { + return; + }; + services.media = media; + + let object_key = format!( + "_meta/{}/{}.json", + claim.request.community_id, + "a".repeat(64) + ); + services + .media + .put(&object_key, b"crash-window", "application/json") + .await + .expect("seed object"); + let current = services + .media + .inspect_current_object(&object_key) + .await + .expect("inspect seeded object"); + let CurrentObjectVersion::Present { size, e_tag, .. } = current else { + panic!("seeded object must be present"); }; let storage = StorageManifest { version: 2, - tenant_keys: vec![object.key.clone()], - tenant_objects: vec![object.clone()], + tenant_keys: vec![object_key.clone()], + tenant_objects: vec![StorageObject { + key: object_key.clone(), + size, + e_tag, + }], git_pointer_keys: Vec::new(), - media_sidecar_keys: Vec::new(), + media_sidecar_keys: vec![object_key.clone()], media_upload_keys: Vec::new(), retained_shared_cas_keys: Vec::new(), unknown_keys: Vec::new(), @@ -1492,7 +1551,7 @@ mod tests { let generation = services.store.fence(&claim.lease).await.expect("fence"); let token = LeaseToken { fence_generation: Some(generation), - ..claim.lease + ..claim.lease.clone() }; services .store @@ -1500,6 +1559,14 @@ mod tests { .await .expect("freeze one-object manifest"); services.store.mark_drained(&token).await.expect("drained"); + + // This is the non-atomic boundary under test: S3 committed the delete, + // then the worker died before checkpoint_storage_object_removed. + services + .media + .delete(&object_key) + .await + .expect("simulate committed delete before crash"); assert!(services .store .completed_storage_object_keys(&token) @@ -1507,17 +1574,18 @@ mod tests { .expect("initial checkpoints") .is_empty()); - assert_eq!( - storage_object_action(&object, &CurrentObjectVersion::Missing) - .expect("missing delete result reconciles"), - StorageObjectAction::AlreadyMissing - ); - let object_key = object.key.clone(); - services - .store - .checkpoint_storage_object_removed(&token, &object_key, true) + let resumed = ClaimedDeletion { + request: services + .store + .get(token.request_id) + .await + .expect("reload drained request"), + lease: claim.lease, + }; + execute_stage(&services, &resumed, &CancellationToken::new()) .await - .expect("checkpoint missing object"); + .expect("production Drained stage reconciles missing first object"); + assert_eq!( services .store @@ -1526,11 +1594,6 @@ mod tests { .expect("completed checkpoints"), std::collections::BTreeSet::from([object_key]) ); - services - .store - .mark_bindings_removed(&token, serde_json::json!({"deleted_keys": 1})) - .await - .expect("advance after reconciliation"); assert_eq!( services.store.get(token.request_id).await.unwrap().stage, DeletionStage::BindingsRemoved From b3388601e33bcd5591afc562affcff21b2b451c1 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 13:55:16 -0700 Subject: [PATCH 11/51] fix: preserve admitted git writes through quiescing Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- .github/workflows/ci.yml | 15 +++++ crates/buzz-db/src/deletion.rs | 74 +++++++++++++++++++++- crates/buzz-db/src/lib.rs | 14 ++-- crates/buzz-deletion/src/lib.rs | 63 +++++++++--------- crates/buzz-relay/src/api/git/transport.rs | 4 +- deploy/charts/buzz/README.md | 7 +- migrations/0028_community_deletion.sql | 33 ++++++++++ schema/schema.sql | 33 ++++++++++ 8 files changed, 195 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e65157705a..544177cad8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,6 +353,7 @@ jobs: cargo nextest archive \ --cargo-profile ci \ -p buzz-db \ + -p buzz-deletion \ -p buzz-relay \ -p buzz-test-client \ --lib \ @@ -684,6 +685,20 @@ jobs: done cat /tmp/buzz-relay.log exit 1 + - name: Community deletion production-path regressions + run: | + cargo nextest run \ + --archive-file target/ci/backend-integration-tests.tar.zst \ + -E '(package(buzz-deletion) and test(/tests::/)) or (package(buzz-relay) and test(/finalize_push_(holds_serving_lease_through_post_cas_publication|db_failure_after_cas_is_not_success_and_releases_lease)/))' \ + --run-ignored ignored-only + env: + DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz + BUZZ_TEST_S3_ENDPOINT: http://localhost:9000 + BUZZ_TEST_S3_ACCESS_KEY: buzz_dev + BUZZ_TEST_S3_SECRET_KEY: buzz_dev_secret + BUZZ_TEST_S3_BUCKET: buzz-media + BUZZ_TEST_S3_REGION: us-east-1 + - name: Invite security tests run: | cargo nextest run \ diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index c9df6d1471..72a52ac2b4 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -1577,6 +1577,60 @@ impl DeletionStore { } } + /// Take the shared community deletion lock inside an existing transaction + /// and authorize a final mutation under an already-admitted serving lease. + /// + /// The lease is checked in the same transaction as the mutation. During + /// quiescing, only this exact unexpired lease and fence generation may + /// finish; active communities continue to accept the admitted write too. + pub async fn guard_transaction_with_serving_lease( + &self, + tx: &mut Transaction<'_, Postgres>, + lease: &ServingWriteLease, + ) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + .bind(lease.community_id.as_uuid()) + .execute(&mut **tx) + .await?; + let valid: bool = sqlx::query_scalar( + "SELECT EXISTS(SELECT 1 FROM community_serving_write_leases lease \ + JOIN communities community ON community.id = lease.community_id \ + WHERE lease.id = $1 AND lease.community_id = $2 AND lease.owner = $3 \ + AND lease.generation = $4 AND lease.fence_generation = $5 \ + AND lease.lease_until >= now() AND community.deleted_at IS NULL \ + AND community.deletion_state IN ('active', 'quiescing') \ + AND community.deletion_fence_generation = lease.fence_generation)", + ) + .bind(lease.id) + .bind(lease.community_id.as_uuid()) + .bind(&lease.owner) + .bind(lease.generation) + .bind(lease.fence_generation) + .fetch_one(&mut **tx) + .await?; + if !valid { + return Err(DbError::AccessDenied(format!( + "stale serving write lease {}", + lease.id + ))); + } + sqlx::query( + "SELECT set_config('buzz.serving_write_community', $1, true), \ + set_config('buzz.serving_write_lease_id', $2, true), \ + set_config('buzz.serving_write_owner', $3, true), \ + set_config('buzz.serving_write_generation', $4, true), \ + set_config('buzz.serving_write_fence_generation', $5, true)", + ) + .bind(lease.community_id.to_string()) + .bind(lease.id.to_string()) + .bind(&lease.owner) + .bind(lease.generation.to_string()) + .bind(lease.fence_generation.to_string()) + .execute(&mut **tx) + .await?; + Ok(()) + } + /// Acquire a durable, expiring lease for an external serving side effect. /// /// The short transaction shares the same advisory lock as the destructive @@ -1929,13 +1983,14 @@ async fn verify_lease( ) -> Result<()> { let valid: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ - WHERE id = $1 AND stage = $2 AND lease_owner = $3 \ + WHERE id = $1 AND community_id = $5 AND stage = $2 AND lease_owner = $3 \ AND lease_generation = $4 AND lease_until >= now() AND blocked_at IS NULL)", ) .bind(token.request_id) .bind(stage.to_string()) .bind(&token.owner) .bind(token.generation) + .bind(token.community_id.as_uuid()) .fetch_one(&mut **tx) .await?; if valid { @@ -1954,7 +2009,8 @@ async fn verify_lease_and_fence( let valid: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM community_deletion_requests request \ JOIN communities community ON community.id = request.community_id \ - WHERE request.id = $1 AND request.stage = $2 AND request.lease_owner = $3 \ + WHERE request.id = $1 AND request.community_id = $6 \ + AND request.stage = $2 AND request.lease_owner = $3 \ AND request.lease_generation = $4 AND request.lease_until >= now() \ AND request.blocked_at IS NULL AND request.fence_generation = $5 \ AND community.deletion_state IN ('fenced', 'tombstone') \ @@ -1965,6 +2021,7 @@ async fn verify_lease_and_fence( .bind(&token.owner) .bind(token.generation) .bind(fence_generation) + .bind(token.community_id.as_uuid()) .fetch_one(&mut **tx) .await?; if valid { @@ -2419,6 +2476,19 @@ mod postgres_tests { store.fence(&stale).await.is_err(), "stale lease must reject" ); + let mut wrong_community = claim.lease.clone(); + wrong_community.community_id = db + .ensure_configured_community(&format!( + "wrong-lease-community-{}.example", + Uuid::new_v4().simple() + )) + .await + .expect("create unrelated community") + .id; + assert!( + store.begin_quiescing(&wrong_community).await.is_err(), + "a lease token must remain bound to its durable request community" + ); store.begin_quiescing(&claim.lease).await.expect("quiesce"); let generation = store.fence(&claim.lease).await.expect("fence"); diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 6b9a5afd98..92a2ac942b 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -1664,19 +1664,19 @@ impl Db { Ok(result) } - /// Insert an event while holding the admitted serving-write lease's - /// community ordering lock through commit. + /// Insert an event while holding and validating an admitted serving-write + /// lease under the community ordering lock through commit. /// /// External side effects use a durable lease rather than one long-lived DB - /// transaction. Their final database mutation must still share the - /// acquire/fence advisory-lock order so a fence cannot commit between the - /// caller's lease verification and this insert. + /// transaction. Their final database mutation presents that exact lease so + /// it may finish during quiescing without admitting any new serving work. pub async fn insert_event_with_serving_write_guard( &self, - community_id: CommunityId, + lease: &deletion::ServingWriteLease, event: &nostr::Event, channel_id: Option, ) -> Result<(StoredEvent, bool)> { + let community_id = lease.community_id; let kind_u16 = event.kind.as_u16(); let kind_u32 = u32::from(kind_u16); if kind_u32 == buzz_core::kind::KIND_AUTH { @@ -1688,7 +1688,7 @@ impl Db { let mut tx = self.pool.begin().await?; self.deletion_store() - .guard_transaction(&mut tx, community_id) + .guard_transaction_with_serving_lease(&mut tx, lease) .await?; let result = event::insert_event_with_thread_metadata_tx( &mut tx, diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 1aab912d07..3281deabf4 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -125,6 +125,11 @@ impl ServingWriteGuard { self.lost.clone() } + /// The durable lease token presented to a final database mutation. + pub fn lease(&self) -> &buzz_db::deletion::ServingWriteLease { + &self.lease + } + /// Release the lease after the side effect completes. pub async fn finish(mut self) -> Result<()> { self.cancel.cancel(); @@ -1379,10 +1384,10 @@ fn print_json(value: &impl Serialize) -> Result<()> { mod tests { use super::*; - async fn claimed_test_deletion(prefix: &str) -> Option<(Services, ClaimedDeletion)> { + async fn claimed_test_deletion(prefix: &str) -> (Services, ClaimedDeletion) { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) - .ok()?; + .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); let pool = sqlx::PgPool::connect(&database_url) .await .expect("connect deletion engine test DB"); @@ -1453,23 +1458,23 @@ mod tests { .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .expect("construct unused Redis pool"), }; - Some((services, claim)) + (services, claim) } - fn deletion_test_media_storage() -> Option> { + fn deletion_test_media_storage() -> Arc { let endpoint = std::env::var("BUZZ_TEST_S3_ENDPOINT") .or_else(|_| std::env::var("BUZZ_S3_ENDPOINT")) - .ok()?; + .expect("BUZZ_TEST_S3_ENDPOINT or BUZZ_S3_ENDPOINT is required"); let access_key = std::env::var("BUZZ_TEST_S3_ACCESS_KEY") .or_else(|_| std::env::var("BUZZ_S3_ACCESS_KEY")) - .ok()?; + .expect("BUZZ_TEST_S3_ACCESS_KEY or BUZZ_S3_ACCESS_KEY is required"); let secret_key = std::env::var("BUZZ_TEST_S3_SECRET_KEY") .or_else(|_| std::env::var("BUZZ_S3_SECRET_KEY")) - .ok()?; + .expect("BUZZ_TEST_S3_SECRET_KEY or BUZZ_S3_SECRET_KEY is required"); let bucket = std::env::var("BUZZ_TEST_S3_BUCKET") .or_else(|_| std::env::var("BUZZ_S3_BUCKET")) - .ok()?; - Some(Arc::new( + .expect("BUZZ_TEST_S3_BUCKET or BUZZ_S3_BUCKET is required"); + Arc::new( MediaStorage::new(&buzz_media::MediaConfig { s3_endpoint: endpoint, s3_access_key: access_key, @@ -1489,7 +1494,7 @@ mod tests { upload_port_header: None, }) .expect("construct deletion test media service"), - )) + ) } #[test] @@ -1500,14 +1505,10 @@ mod tests { } #[tokio::test] + #[ignore = "requires Postgres and S3-compatible storage"] async fn drained_stage_reconciles_first_object_deleted_before_checkpoint() { - let Some((mut services, claim)) = claimed_test_deletion("deletion-first-missing").await - else { - return; - }; - let Some(media) = deletion_test_media_storage() else { - return; - }; + let (mut services, claim) = claimed_test_deletion("deletion-first-missing").await; + let media = deletion_test_media_storage(); services.media = media; let object_key = format!( @@ -1667,10 +1668,9 @@ mod tests { } #[tokio::test] + #[ignore = "requires Postgres"] async fn final_storage_verification_rejects_late_target_binding() { - let Some((services, claim)) = claimed_test_deletion("deletion-late-binding").await else { - return; - }; + let (services, claim) = claimed_test_deletion("deletion-late-binding").await; let community = claim.request.community_id; let late_key = format!("_meta/{community}/{}.json", "a".repeat(64)); let error = verify_storage_absence_from_objects( @@ -1689,10 +1689,9 @@ mod tests { } #[tokio::test] + #[ignore = "requires Postgres"] async fn stale_lease_during_failure_recording_is_lost_ownership() { - let Some((services, claim)) = claimed_test_deletion("deletion-stale-record").await else { - return; - }; + let (services, claim) = claimed_test_deletion("deletion-stale-record").await; let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) .expect("test database URL"); @@ -1718,13 +1717,11 @@ mod tests { } #[tokio::test] + #[ignore = "requires Postgres"] async fn serving_guard_cancels_protected_operation_when_heartbeat_is_lost() { - let database_url = match std::env::var("BUZZ_TEST_DATABASE_URL") + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) - { - Ok(url) => url, - Err(_) => return, - }; + .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); let pool = sqlx::PgPool::connect(&database_url) .await .expect("connect serving guard test DB"); @@ -1764,10 +1761,9 @@ mod tests { } #[tokio::test] + #[ignore = "requires Postgres"] async fn guarded_external_step_rejects_preexisting_heartbeat_loss_without_polling_operation() { - let Some((services, claim)) = claimed_test_deletion("deletion-heartbeat").await else { - return; - }; + let (services, claim) = claimed_test_deletion("deletion-heartbeat").await; let heartbeat_lost = CancellationToken::new(); heartbeat_lost.cancel(); let polled = Arc::new(AtomicBool::new(false)); @@ -1798,10 +1794,9 @@ mod tests { } #[tokio::test] + #[ignore = "requires Postgres"] async fn shutdown_during_stage_releases_claim_without_recording_retry() { - let Some((services, claim)) = claimed_test_deletion("deletion-shutdown").await else { - return; - }; + let (services, claim) = claimed_test_deletion("deletion-shutdown").await; let request_id = claim.request.id; let retry_count = claim.request.retry_count; let shutdown = CancellationToken::new(); diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index b01b4561b5..fc0b774bfa 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -2018,13 +2018,13 @@ async fn finalize_push_inner( } else { state .db - .insert_event_with_serving_write_guard(ctx.tenant.community(), &event, None) + .insert_event_with_serving_write_guard(serving_write.lease(), &event, None) .await }; #[cfg(not(test))] let insert_result = state .db - .insert_event_with_serving_write_guard(ctx.tenant.community(), &event, None) + .insert_event_with_serving_write_guard(serving_write.lease(), &event, None) .await; match insert_result { Ok((stored, true)) => { diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index 2ecc5d4440..9ec31282ff 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -201,9 +201,10 @@ Track application p99 for events/media/Git/workflows alongside the lease gauges; investigate rising dead tuples or expired rows before raising worker scale. Object removal is checkpointed per key and each attempt deletes at most -`deletionWorker.storageDeleteBatchSize` (default 100). After recorded deletion -progress, missing keys resume as already-complete; a missing key before any -checkpoint still blocks as unexplained drift. Changed/versioned keys fail closed. +`deletionWorker.storageDeleteBatchSize` (default 100). Deletion is resumable +from durable checkpoints. Missing keys are recorded as `already_missing` and +processing continues, including the crash window where a DELETE committed +before its first PostgreSQL checkpoint. Changed/versioned keys fail closed. Final verification performs a fresh complete target inventory before advancing. ## Device pairing relay diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 72e99d1a2f..1b074ecf4d 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -161,6 +161,12 @@ DECLARE generation BIGINT; executor_community TEXT; executor_generation TEXT; + serving_community TEXT; + serving_lease_id TEXT; + serving_owner TEXT; + serving_generation TEXT; + serving_fence_generation TEXT; + serving_lease_valid BOOLEAN := false; BEGIN -- Nullable operator-attribution rows without a tenant are unrelated. IF target IS NULL THEN @@ -186,6 +192,33 @@ BEGIN RETURN; END IF; + -- A serving mutation admitted before quiescing may finish only while its + -- exact durable lease remains current and bound to this fence generation. + serving_community := current_setting('buzz.serving_write_community', true); + serving_lease_id := current_setting('buzz.serving_write_lease_id', true); + serving_owner := current_setting('buzz.serving_write_owner', true); + serving_generation := current_setting('buzz.serving_write_generation', true); + serving_fence_generation := current_setting('buzz.serving_write_fence_generation', true); + IF lifecycle IN ('active', 'quiescing') + AND serving_community = target::TEXT + AND serving_lease_id ~ '^[0-9a-fA-F-]{36}$' + AND serving_generation ~ '^[0-9]+$' + AND serving_fence_generation ~ '^[0-9]+$' + AND serving_fence_generation::BIGINT = generation THEN + SELECT EXISTS( + SELECT 1 FROM community_serving_write_leases lease + WHERE lease.id = serving_lease_id::UUID + AND lease.community_id = target + AND lease.owner = serving_owner + AND lease.generation = serving_generation::BIGINT + AND lease.fence_generation = serving_fence_generation::BIGINT + AND lease.lease_until >= now() + ) INTO serving_lease_valid; + IF serving_lease_valid THEN + RETURN; + END IF; + END IF; + IF lifecycle <> 'active' THEN RAISE EXCEPTION 'community write fenced: community % generation %', target, generation USING ERRCODE = 'object_not_in_prerequisite_state'; diff --git a/schema/schema.sql b/schema/schema.sql index 902033b1a4..33a5657471 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1220,6 +1220,12 @@ DECLARE generation BIGINT; executor_community TEXT; executor_generation TEXT; + serving_community TEXT; + serving_lease_id TEXT; + serving_owner TEXT; + serving_generation TEXT; + serving_fence_generation TEXT; + serving_lease_valid BOOLEAN := false; BEGIN -- Nullable operator-attribution rows without a tenant are unrelated. IF target IS NULL THEN @@ -1245,6 +1251,33 @@ BEGIN RETURN; END IF; + -- A serving mutation admitted before quiescing may finish only while its + -- exact durable lease remains current and bound to this fence generation. + serving_community := current_setting('buzz.serving_write_community', true); + serving_lease_id := current_setting('buzz.serving_write_lease_id', true); + serving_owner := current_setting('buzz.serving_write_owner', true); + serving_generation := current_setting('buzz.serving_write_generation', true); + serving_fence_generation := current_setting('buzz.serving_write_fence_generation', true); + IF lifecycle IN ('active', 'quiescing') + AND serving_community = target::TEXT + AND serving_lease_id ~ '^[0-9a-fA-F-]{36}$' + AND serving_generation ~ '^[0-9]+$' + AND serving_fence_generation ~ '^[0-9]+$' + AND serving_fence_generation::BIGINT = generation THEN + SELECT EXISTS( + SELECT 1 FROM community_serving_write_leases lease + WHERE lease.id = serving_lease_id::UUID + AND lease.community_id = target + AND lease.owner = serving_owner + AND lease.generation = serving_generation::BIGINT + AND lease.fence_generation = serving_fence_generation::BIGINT + AND lease.lease_until >= now() + ) INTO serving_lease_valid; + IF serving_lease_valid THEN + RETURN; + END IF; + END IF; + IF lifecycle <> 'active' THEN RAISE EXCEPTION 'community write fenced: community % generation %', target, generation USING ERRCODE = 'object_not_in_prerequisite_state'; From 7392b3cdce40f69b341d651881992f95a5670c0d Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 14:47:25 -0700 Subject: [PATCH 12/51] fix: bind deletion approval to frozen inventory digest Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- .github/workflows/ci.yml | 2 +- crates/buzz-db/src/deletion.rs | 34 ++++++++++++++++++++++++++ migrations/0028_community_deletion.sql | 10 +++++--- schema/schema.sql | 10 +++++--- 4 files changed, 49 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 544177cad8..b3a4d37eac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -689,7 +689,7 @@ jobs: run: | cargo nextest run \ --archive-file target/ci/backend-integration-tests.tar.zst \ - -E '(package(buzz-deletion) and test(/tests::/)) or (package(buzz-relay) and test(/finalize_push_(holds_serving_lease_through_post_cas_publication|db_failure_after_cas_is_not_success_and_releases_lease)/))' \ + -E '(package(buzz-deletion) and test(/tests::/)) or (package(buzz-db) and test(/deletion::postgres_tests/)) or (package(buzz-relay) and test(/finalize_push_(holds_serving_lease_through_post_cas_publication|db_failure_after_cas_is_not_success_and_releases_lease)/))' \ --run-ignored ignored-only env: DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 72a52ac2b4..713c5d5447 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -2440,6 +2440,18 @@ mod postgres_tests { .expect("live schema after row churn"); assert_eq!(current_schema, inventory.schema); + let mismatched_insert = sqlx::query( + "INSERT INTO community_deletion_approvals \ + (request_id, inventory_digest, approved_by) VALUES ($1, $2, 'tampered')", + ) + .bind(request.id) + .bind(vec![0_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_insert.is_err(), + "a mismatched approval must be unrepresentable" + ); let approved = store .approve(request.id, "approver-a", Some("reviewed")) .await @@ -2449,6 +2461,28 @@ mod postgres_tests { approved.inventory_digest, Some(hex::encode(inventory.digest().unwrap())) ); + let mismatched_approval = sqlx::query( + "UPDATE community_deletion_approvals SET inventory_digest = $2 WHERE request_id = $1", + ) + .bind(request.id) + .bind(vec![0_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_approval.is_err(), + "approval digest must remain database-bound to the frozen request digest" + ); + let mismatched_request = sqlx::query( + "UPDATE community_deletion_requests SET inventory_digest = $2 WHERE id = $1", + ) + .bind(request.id) + .bind(vec![1_u8; 32]) + .execute(&db.pool) + .await; + assert!( + mismatched_request.is_err(), + "the frozen request digest must remain bound to its approval" + ); assert!(store .claim_specific(request.id, "executor-a", DEFAULT_LEASE_DURATION) .await diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 1b074ecf4d..7d45894782 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -49,7 +49,8 @@ CREATE TABLE community_deletion_requests ( updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), completed_at TIMESTAMPTZ, CHECK ((blocked_at IS NULL) = (blocked_reason IS NULL)), - CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)) + CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)), + UNIQUE (id, inventory_digest) ); CREATE INDEX community_deletion_requests_runnable ON community_deletion_requests (next_attempt_at, created_at) @@ -61,11 +62,14 @@ CREATE INDEX community_deletion_requests_lease WHERE lease_owner IS NOT NULL; CREATE TABLE community_deletion_approvals ( - request_id UUID PRIMARY KEY REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + request_id UUID PRIMARY KEY, inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), approved_by TEXT NOT NULL, note TEXT, - approved_at TIMESTAMPTZ NOT NULL DEFAULT now() + approved_at TIMESTAMPTZ NOT NULL DEFAULT now(), + FOREIGN KEY (request_id, inventory_digest) + REFERENCES community_deletion_requests(id, inventory_digest) + ON DELETE RESTRICT ); CREATE TABLE community_deletion_checkpoints ( diff --git a/schema/schema.sql b/schema/schema.sql index 33a5657471..19da45c5a7 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1118,7 +1118,8 @@ CREATE TABLE community_deletion_requests ( updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), completed_at TIMESTAMPTZ, CHECK ((blocked_at IS NULL) = (blocked_reason IS NULL)), - CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)) + CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)), + UNIQUE (id, inventory_digest) ); CREATE INDEX community_deletion_requests_runnable ON community_deletion_requests (next_attempt_at, created_at) @@ -1129,11 +1130,14 @@ CREATE INDEX community_deletion_requests_lease ON community_deletion_requests (lease_until) WHERE lease_owner IS NOT NULL; CREATE TABLE community_deletion_approvals ( - request_id UUID PRIMARY KEY REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, + request_id UUID PRIMARY KEY, inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), approved_by TEXT NOT NULL, note TEXT, - approved_at TIMESTAMPTZ NOT NULL DEFAULT now() + approved_at TIMESTAMPTZ NOT NULL DEFAULT now(), + FOREIGN KEY (request_id, inventory_digest) + REFERENCES community_deletion_requests(id, inventory_digest) + ON DELETE RESTRICT ); CREATE TABLE community_deletion_checkpoints ( request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, From ebcc9a57b47c03fd3fc0df7db86ed4cf9210f0e0 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 15:49:18 -0700 Subject: [PATCH 13/51] fix: bind community deletion execution to approval Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- Cargo.lock | 1 + crates/buzz-db/src/deletion.rs | 192 +++++++++++++++++++++---- crates/buzz-db/src/migration.rs | 68 ++------- crates/buzz-deletion/Cargo.toml | 1 + crates/buzz-deletion/src/lib.rs | 92 ++++++++++-- migrations/0028_community_deletion.sql | 55 ++++++- schema/schema.sql | 53 ++++++- 7 files changed, 354 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62a7cef6be..95e26b150d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1031,6 +1031,7 @@ dependencies = [ "buzz-media", "clap", "deadpool-redis", + "hex", "redis", "serde", "serde_json", diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 713c5d5447..bc82a03b95 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -836,23 +836,34 @@ impl DeletionStore { note: Option<&str>, ) -> Result { let mut tx = self.pool.begin().await?; - let digest: Vec = sqlx::query_scalar( - "SELECT inventory_digest FROM community_deletion_requests \ - WHERE id = $1 AND stage = 'inventoried' AND blocked_at IS NULL FOR UPDATE", - ) - .bind(request_id) - .fetch_optional(&mut *tx) - .await? - .ok_or_else(|| { - DbError::DeletionSafety(format!( - "deletion {request_id} is not an unblocked inventoried request" - )) - })?; + let (community_id, digest, inventory_manifest): (Uuid, Vec, serde_json::Value) = + sqlx::query_as( + "SELECT community_id, inventory_digest, inventory_manifest \ + FROM community_deletion_requests \ + WHERE id = $1 AND stage = 'inventoried' AND blocked_at IS NULL FOR UPDATE", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} is not an unblocked inventoried request" + )) + })?; + let inventory: FrozenInventory = serde_json::from_value(inventory_manifest)?; + let recomputed_digest = inventory.digest()?; + if digest.as_slice() != recomputed_digest { + return Err(DbError::DeletionSafety(format!( + "deletion {request_id} frozen inventory digest does not match its manifest" + ))); + } sqlx::query( "INSERT INTO community_deletion_approvals \ - (request_id, inventory_digest, approved_by, note) VALUES ($1, $2, $3, $4)", + (request_id, community_id, inventory_digest, approved_by, note) \ + VALUES ($1, $2, $3, $4, $5)", ) .bind(request_id) + .bind(community_id) .bind(&digest) .bind(approved_by) .bind(note) @@ -902,15 +913,19 @@ impl DeletionStore { let row = sqlx::query( r#" WITH candidate AS ( - SELECT id - FROM community_deletion_requests - WHERE ($1::uuid IS NULL OR id = $1) - AND stage IN ('approved', 'fenced', 'drained', 'bindings_removed', + SELECT request.id + FROM community_deletion_requests request + JOIN community_deletion_approvals approval + ON approval.request_id = request.id + AND approval.community_id = request.community_id + AND approval.inventory_digest = request.inventory_digest + WHERE ($1::uuid IS NULL OR request.id = $1) + AND request.stage IN ('approved', 'fenced', 'drained', 'bindings_removed', 'postgres_purged', 'cache_purged', 'logically_verified') - AND blocked_at IS NULL - AND next_attempt_at <= now() - AND (lease_until IS NULL OR lease_until < now()) - ORDER BY created_at, id + AND request.blocked_at IS NULL + AND request.next_attempt_at <= now() + AND (request.lease_until IS NULL OR request.lease_until < now()) + ORDER BY request.created_at, request.id FOR UPDATE SKIP LOCKED LIMIT 1 ) @@ -971,10 +986,17 @@ impl DeletionStore { let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); let mut tx = self.pool.begin().await?; let affected = sqlx::query( - "UPDATE community_deletion_requests \ + "UPDATE community_deletion_requests request \ SET lease_until = now() + make_interval(secs => $4), updated_at = now() \ - WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3 \ - AND lease_until >= now()", + WHERE request.id = $1 AND request.lease_owner = $2 \ + AND request.lease_generation = $3 AND request.lease_until >= now() \ + AND request.blocked_at IS NULL \ + AND request.stage IN ('approved', 'fenced', 'drained', 'bindings_removed', \ + 'postgres_purged', 'cache_purged', 'logically_verified') \ + AND EXISTS (SELECT 1 FROM community_deletion_approvals approval \ + WHERE approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest)", ) .bind(token.request_id) .bind(&token.owner) @@ -1982,9 +2004,13 @@ async fn verify_lease( stage: DeletionStage, ) -> Result<()> { let valid: bool = sqlx::query_scalar( - "SELECT EXISTS(SELECT 1 FROM community_deletion_requests \ - WHERE id = $1 AND community_id = $5 AND stage = $2 AND lease_owner = $3 \ - AND lease_generation = $4 AND lease_until >= now() AND blocked_at IS NULL)", + "SELECT EXISTS(SELECT 1 FROM community_deletion_requests request \ + JOIN community_deletion_approvals approval ON approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest \ + WHERE request.id = $1 AND request.community_id = $5 AND request.stage = $2 \ + AND request.lease_owner = $3 AND request.lease_generation = $4 \ + AND request.lease_until >= now() AND request.blocked_at IS NULL)", ) .bind(token.request_id) .bind(stage.to_string()) @@ -2009,6 +2035,9 @@ async fn verify_lease_and_fence( let valid: bool = sqlx::query_scalar( "SELECT EXISTS(SELECT 1 FROM community_deletion_requests request \ JOIN communities community ON community.id = request.community_id \ + JOIN community_deletion_approvals approval ON approval.request_id = request.id \ + AND approval.community_id = request.community_id \ + AND approval.inventory_digest = request.inventory_digest \ WHERE request.id = $1 AND request.community_id = $6 \ AND request.stage = $2 AND request.lease_owner = $3 \ AND request.lease_generation = $4 AND request.lease_until >= now() \ @@ -2336,7 +2365,7 @@ mod tests { let inventory = FrozenInventory { schema: SchemaManifest { revision: 1, - migration_version: 27, + migration_version: EXPECTED_MIGRATION_VERSION, scoped_tables: vec!["events".to_string()], row_counts: BTreeMap::from([("events".to_string(), 3)]), fenced_tables: vec!["events".to_string()], @@ -2442,9 +2471,11 @@ mod postgres_tests { let mismatched_insert = sqlx::query( "INSERT INTO community_deletion_approvals \ - (request_id, inventory_digest, approved_by) VALUES ($1, $2, 'tampered')", + (request_id, community_id, inventory_digest, approved_by) \ + VALUES ($1, $2, $3, 'tampered')", ) .bind(request.id) + .bind(*request.community_id.as_uuid()) .bind(vec![0_u8; 32]) .execute(&db.pool) .await; @@ -2490,6 +2521,109 @@ mod postgres_tests { .is_some()); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn approved_request_cannot_be_retargeted_rewritten_or_claimed_without_approval() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + let other_host = format!("control-{}.example", Uuid::new_v4().simple()); + let control = db + .ensure_configured_community(&other_host) + .await + .expect("create control community"); + + for mutation in [ + sqlx::query("UPDATE community_deletion_requests SET community_id = $2 WHERE id = $1") + .bind(request.id) + .bind(*control.id.as_uuid()) + .execute(&db.pool) + .await, + sqlx::query("UPDATE community_deletion_requests SET community_host = $2 WHERE id = $1") + .bind(request.id) + .bind(&other_host) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests SET inventory_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests SET storage_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + ] { + assert!(mutation.is_err(), "frozen deletion target and inventory must be immutable"); + } + + store + .approve(request.id, "approver", None) + .await + .expect("approve request"); + let claim = store + .claim_specific(request.id, "forged-executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim approved request") + .expect("approved request is claimable"); + let approval_delete = + sqlx::query("DELETE FROM community_deletion_approvals WHERE request_id = $1") + .bind(request.id) + .execute(&db.pool) + .await; + assert!( + approval_delete.is_err(), + "approval evidence must be immutable" + ); + for approval_update in [ + "UPDATE community_deletion_approvals SET approved_by = 'forged' WHERE request_id = $1", + "UPDATE community_deletion_approvals SET approved_at = now() + interval '1 hour' WHERE request_id = $1", + "UPDATE community_deletion_approvals SET note = 'rewritten' WHERE request_id = $1", + ] { + assert!( + sqlx::query(approval_update) + .bind(request.id) + .execute(&db.pool) + .await + .is_err(), + "approval evidence updates must be rejected" + ); + } + store + .verify_execution_token(&claim.lease, DeletionStage::Approved) + .await + .expect("matching approval keeps lease valid"); + sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = now(), blocked_reason = 'operator hold' WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await + .expect("block claimed request"); + assert!( + store + .heartbeat(&claim.lease, "worker", DEFAULT_LEASE_DURATION, false,) + .await + .is_err(), + "blocked requests must not renew destructive leases" + ); + + let (forged, _) = inventoried_request(&db, &store).await; + sqlx::query("UPDATE community_deletion_requests SET stage = 'approved' WHERE id = $1") + .bind(forged.id) + .execute(&db.pool) + .await + .expect("forge runnable stage without approval"); + assert!(store + .claim_specific(forged.id, "forged-executor-2", DEFAULT_LEASE_DURATION) + .await + .expect("claim forged request") + .is_none()); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn stale_claim_and_fence_generation_fail_closed() { diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index cbb8e0e062..2f340e0715 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -1027,11 +1027,7 @@ mod tests { assert!(heartbeat.contains("_operator_global_tables")); - // Channel-id lookup index (0027): serves the tenant-independent - // `channels` lookups that carry no community_id predicate, which no - // community_id-leading index can satisfy. Covering + partial so the - // planner can go index-only; asserted NOT UNIQUE because `id` alone is - // not unique across communities. + // Channel-id lookup index (0027). let channel_id_index = migrations .iter() .find(|migration| migration.sql.as_str().contains("idx_channels_id_live")) @@ -1042,7 +1038,7 @@ mod tests { assert!(!channel_id_index.sql.as_str().contains("CREATE UNIQUE INDEX")); assert!(desired_schema.contains("idx_channels_id_live")); - // Main already owns migration 0028 for long reaction payloads. + // Main owns 0028 for long reaction payloads. let long_reactions = migrations .iter() .find(|migration| { @@ -1055,8 +1051,8 @@ mod tests { assert_eq!(long_reactions.version, 28); assert!(desired_schema.contains("emoji VARCHAR(66) NOT NULL")); - // Durable whole-community deletion control plane. Its final migration - // number is reconciled after the branch is replayed onto current main. + // Durable whole-community deletion control plane. Final numbering is + // reconciled once all branch commits have replayed onto current main. let deletion = migrations .iter() .find(|migration| { @@ -1066,52 +1062,9 @@ mod tests { .contains("CREATE TABLE community_deletion_requests") }) .expect("community-deletion migration"); - assert!(deletion.sql.as_str().contains("CREATE TABLE community_deletion_approvals")); - assert!(deletion.sql.as_str().contains("CREATE TABLE community_deletion_checkpoints")); - assert!(deletion.sql.as_str().contains("CREATE FUNCTION enforce_community_write_fence")); - assert!(deletion.sql.as_str().contains("CREATE FUNCTION enforce_community_tombstone")); - assert!(deletion.sql.as_str().contains("community tombstones are permanent")); - - // Channel-id lookup index (0027): serves the tenant-independent - // `channels` lookups that carry no community_id predicate, which no - // community_id-leading index can satisfy. Covering + partial so the - // planner can go index-only; asserted NOT UNIQUE because `id` alone is - // not unique in this table (the same channel id may exist under more - // than one community), so a unique index would encode a false - // constraint and fail to build on such a database. - assert_eq!(migrations[26].version, 27); - let channel_id_index = migrations[26].sql.as_str(); - assert!(channel_id_index.contains("idx_channels_id_live")); - assert!(channel_id_index.contains("INCLUDE (community_id)")); - assert!(channel_id_index.contains("WHERE deleted_at IS NULL")); - assert!( - !channel_id_index.contains("CREATE UNIQUE INDEX"), - "channels.id is not unique across communities — index must not be UNIQUE", - ); - assert!( - desired_schema.contains("idx_channels_id_live"), - "desired-state schema must carry the channel-id lookup index", - ); -<<<<<<< HEAD - - assert_eq!(migrations[27].version, 28); - let long_reactions = migrations[27].sql.as_str(); - assert!( - long_reactions.contains("ALTER TABLE reactions ALTER COLUMN emoji TYPE VARCHAR(66)") - ); - assert!(desired_schema.contains("emoji VARCHAR(66) NOT NULL")); -||||||| parent of 037fcbdab (feat: add durable community deletion worker) -======= -||||||| parent of d04e4b503 (feat: add durable community deletion worker) -======= - - // Durable whole-community deletion control plane and universal DB fence. - assert_eq!(migrations[26].version, 27); - let deletion = migrations[26].sql.as_str(); - assert!(deletion.contains("CREATE TABLE community_deletion_requests")); + let deletion = deletion.sql.as_str(); assert!(deletion.contains("CREATE TABLE community_deletion_approvals")); assert!(deletion.contains("CREATE TABLE community_deletion_checkpoints")); - assert!(deletion.contains("CREATE TABLE community_deletion_retention_exceptions")); assert!(deletion.contains("CREATE TABLE community_serving_write_leases")); assert!(deletion.contains("CREATE TABLE community_deletion_executor_heartbeats")); assert!(deletion.contains("CREATE FUNCTION assert_community_write_allowed")); @@ -1121,11 +1074,10 @@ mod tests { assert!(deletion.contains("CREATE FUNCTION enforce_community_tombstone")); assert!(deletion.contains("community tombstones are permanent")); assert!(deletion.contains("SET LOCAL lock_timeout = '5s'")); - assert!(deletion.contains("'active', 'quiescing', 'fenced', 'tombstone'")); - assert!(deletion.contains("_operator_global_tables")); - assert!(deletion.contains("'submitted', 'inventoried', 'approved', 'fenced', 'drained'")); ->>>>>>> d04e4b503 (feat: add durable community deletion worker) ->>>>>>> 037fcbdab (feat: add durable community deletion worker) + assert!(deletion.contains("UNIQUE (id, community_id, inventory_digest)")); + assert!(deletion.contains("FOREIGN KEY (request_id, community_id, inventory_digest)")); + assert!(deletion.contains("prevent_community_deletion_request_retargeting")); + assert!(deletion.contains("prevent_community_deletion_approval_removal")); } #[test] @@ -1133,7 +1085,7 @@ mod tests { let violations = migrations_missing_community_write_fence_attachment(); assert!( violations.is_empty(), - "migrations after 0027 must explicitly attach every new community write fence:\n{}", + "migrations after 0028 must explicitly attach every new community write fence:\n{}", violations.join("\n") ); } diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml index 0da39834a3..9a3593b665 100644 --- a/crates/buzz-deletion/Cargo.toml +++ b/crates/buzz-deletion/Cargo.toml @@ -16,6 +16,7 @@ buzz-db = { workspace = true } buzz-media = { workspace = true } clap = { version = "4", features = ["derive"] } deadpool-redis = { workspace = true } +hex = { workspace = true } redis = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 3281deabf4..f4b2f64d82 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -564,6 +564,46 @@ fn storage_object_action( } } +fn validate_frozen_inventory(request: &DeletionRequest) -> Result { + let frozen: FrozenInventory = serde_json::from_value( + request + .inventory_manifest + .clone() + .ok_or_else(|| permanent("approved request has no frozen inventory"))?, + ) + .map_err(permanent_source)?; + let expected_digest = request + .inventory_digest + .as_deref() + .ok_or_else(|| permanent("approved request has no frozen inventory digest"))?; + let actual_digest = hex::encode(frozen.digest().map_err(permanent_source)?); + if actual_digest != expected_digest { + return Err(permanent("approved frozen inventory digest mismatch")); + } + validate_storage_ownership(request, &frozen.storage)?; + Ok(frozen) +} + +fn validate_storage_ownership(request: &DeletionRequest, manifest: &StorageManifest) -> Result<()> { + buzz_db::deletion::validate_storage_manifest(manifest)?; + let classified = deletion_inventory( + *request.community_id.as_uuid(), + manifest + .tenant_objects + .iter() + .map(|object| (object.key.clone(), object.size)), + ); + if classified.tenant_keys() != manifest.tenant_keys + || !classified.unknown_keys.is_empty() + || !classified.retained_shared_cas_keys.is_empty() + { + return Err(permanent( + "storage manifest contains keys not owned by the deletion target", + )); + } + Ok(()) +} + async fn build_inventory( services: &Services, request: &DeletionRequest, @@ -952,13 +992,7 @@ async fn execute_stage( .store .inventory_schema(request.community_id) .await?; - let frozen: FrozenInventory = serde_json::from_value( - request - .inventory_manifest - .clone() - .ok_or_else(|| permanent("approved request has no frozen inventory"))?, - ) - .map_err(permanent_source)?; + let frozen = validate_frozen_inventory(request)?; if live_schema != frozen.schema { return Err(permanent( "approved structural catalog drifted before fencing", @@ -1011,7 +1045,7 @@ async fn execute_stage( manifest } }; - buzz_db::deletion::validate_storage_manifest(&destructive)?; + validate_storage_ownership(request, &destructive)?; if services .store .serving_writes_drained(request.community_id) @@ -1029,7 +1063,7 @@ async fn execute_stage( .clone() .context("request has no post-fence destructive storage manifest")?, )?; - buzz_db::deletion::validate_storage_manifest(&storage)?; + validate_storage_ownership(request, &storage)?; let completed = services.store.completed_storage_object_keys(&token).await?; let mut processed = 0usize; for object in storage @@ -1128,12 +1162,7 @@ async fn execute_stage( .await?; } DeletionStage::LogicallyVerified => { - let frozen: FrozenInventory = serde_json::from_value( - request - .inventory_manifest - .clone() - .context("request has no frozen inventory")?, - )?; + let frozen = validate_frozen_inventory(request)?; services .store .mark_retention_pending( @@ -1504,6 +1533,39 @@ mod tests { assert_eq!(keys, STORAGE_DELETE_BATCH_SIZE); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn frozen_inventory_digest_and_storage_ownership_fail_closed() { + let (_, claim) = claimed_test_deletion("deletion-integrity").await; + assert!(validate_frozen_inventory(&claim.request).is_ok()); + + let mut digest_tampered = claim.request.clone(); + digest_tampered.inventory_manifest = Some(serde_json::json!({ + "schema": {"revision": 0, "migration_version": 0, "scoped_tables": [], "row_counts": {}, "fenced_tables": []}, + "storage": {"version": 2, "tenant_keys": [], "tenant_objects": [], "git_pointer_keys": [], "media_sidecar_keys": [], "media_upload_keys": [], "retained_shared_cas_keys": [], "unknown_keys": [], "unsupported_version_keys": []} + })); + assert!(validate_frozen_inventory(&digest_tampered).is_err()); + + let foreign_community = Uuid::new_v4(); + let foreign_key = format!("_meta/{foreign_community}/{}.json", "a".repeat(64)); + let foreign_manifest = StorageManifest { + version: 2, + tenant_keys: vec![foreign_key.clone()], + tenant_objects: vec![StorageObject { + key: foreign_key, + size: 1, + e_tag: Some("etag".to_string()), + }], + git_pointer_keys: Vec::new(), + media_sidecar_keys: Vec::new(), + media_upload_keys: Vec::new(), + retained_shared_cas_keys: Vec::new(), + unknown_keys: Vec::new(), + unsupported_version_keys: Vec::new(), + }; + assert!(validate_storage_ownership(&claim.request, &foreign_manifest).is_err()); + } + #[tokio::test] #[ignore = "requires Postgres and S3-compatible storage"] async fn drained_stage_reconciles_first_object_deleted_before_checkpoint() { diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 7d45894782..3b909ac5be 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -50,7 +50,7 @@ CREATE TABLE community_deletion_requests ( completed_at TIMESTAMPTZ, CHECK ((blocked_at IS NULL) = (blocked_reason IS NULL)), CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)), - UNIQUE (id, inventory_digest) + UNIQUE (id, community_id, inventory_digest) ); CREATE INDEX community_deletion_requests_runnable ON community_deletion_requests (next_attempt_at, created_at) @@ -63,15 +63,64 @@ CREATE INDEX community_deletion_requests_lease CREATE TABLE community_deletion_approvals ( request_id UUID PRIMARY KEY, + community_id UUID NOT NULL, inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), approved_by TEXT NOT NULL, note TEXT, approved_at TIMESTAMPTZ NOT NULL DEFAULT now(), - FOREIGN KEY (request_id, inventory_digest) - REFERENCES community_deletion_requests(id, inventory_digest) + FOREIGN KEY (request_id, community_id, inventory_digest) + REFERENCES community_deletion_requests(id, community_id, inventory_digest) ON DELETE RESTRICT ); +-- The approval identity is only meaningful while its frozen target and +-- inventory remain unchanged. Make those facts irreversible in the database, +-- not merely conventions in the worker. +CREATE FUNCTION prevent_community_deletion_request_retargeting() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.community_id IS DISTINCT FROM OLD.community_id + OR NEW.community_host IS DISTINCT FROM OLD.community_host + THEN + RAISE EXCEPTION 'community deletion target identity is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF OLD.inventory_frozen_at IS NOT NULL AND ( + NEW.schema_manifest IS DISTINCT FROM OLD.schema_manifest + OR NEW.storage_manifest IS DISTINCT FROM OLD.storage_manifest + OR NEW.inventory_manifest IS DISTINCT FROM OLD.inventory_manifest + OR NEW.inventory_digest IS DISTINCT FROM OLD.inventory_digest + OR NEW.inventory_frozen_at IS DISTINCT FROM OLD.inventory_frozen_at + ) THEN + RAISE EXCEPTION 'frozen community deletion inventory is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER community_deletion_request_retargeting_guard +BEFORE UPDATE ON community_deletion_requests +FOR EACH ROW +EXECUTE FUNCTION prevent_community_deletion_request_retargeting(); + +CREATE FUNCTION prevent_community_deletion_approval_removal() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE EXCEPTION 'community deletion approval evidence is immutable' + USING ERRCODE = 'integrity_constraint_violation'; +END; +$$; + +CREATE TRIGGER community_deletion_approval_removal_guard +BEFORE UPDATE OR DELETE ON community_deletion_approvals +FOR EACH ROW +EXECUTE FUNCTION prevent_community_deletion_approval_removal(); + CREATE TABLE community_deletion_checkpoints ( request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, sequence BIGINT GENERATED ALWAYS AS IDENTITY, diff --git a/schema/schema.sql b/schema/schema.sql index 19da45c5a7..27bdea1e78 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1119,7 +1119,7 @@ CREATE TABLE community_deletion_requests ( completed_at TIMESTAMPTZ, CHECK ((blocked_at IS NULL) = (blocked_reason IS NULL)), CHECK ((inventory_frozen_at IS NULL) = (inventory_digest IS NULL)), - UNIQUE (id, inventory_digest) + UNIQUE (id, community_id, inventory_digest) ); CREATE INDEX community_deletion_requests_runnable ON community_deletion_requests (next_attempt_at, created_at) @@ -1131,14 +1131,61 @@ CREATE INDEX community_deletion_requests_lease CREATE TABLE community_deletion_approvals ( request_id UUID PRIMARY KEY, + community_id UUID NOT NULL, inventory_digest BYTEA NOT NULL CHECK (length(inventory_digest) = 32), approved_by TEXT NOT NULL, note TEXT, approved_at TIMESTAMPTZ NOT NULL DEFAULT now(), - FOREIGN KEY (request_id, inventory_digest) - REFERENCES community_deletion_requests(id, inventory_digest) + FOREIGN KEY (request_id, community_id, inventory_digest) + REFERENCES community_deletion_requests(id, community_id, inventory_digest) ON DELETE RESTRICT ); + +CREATE FUNCTION prevent_community_deletion_request_retargeting() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.community_id IS DISTINCT FROM OLD.community_id + OR NEW.community_host IS DISTINCT FROM OLD.community_host + THEN + RAISE EXCEPTION 'community deletion target identity is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + IF OLD.inventory_frozen_at IS NOT NULL AND ( + NEW.schema_manifest IS DISTINCT FROM OLD.schema_manifest + OR NEW.storage_manifest IS DISTINCT FROM OLD.storage_manifest + OR NEW.inventory_manifest IS DISTINCT FROM OLD.inventory_manifest + OR NEW.inventory_digest IS DISTINCT FROM OLD.inventory_digest + OR NEW.inventory_frozen_at IS DISTINCT FROM OLD.inventory_frozen_at + ) THEN + RAISE EXCEPTION 'frozen community deletion inventory is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; +END; +$$; + +CREATE TRIGGER community_deletion_request_retargeting_guard +BEFORE UPDATE ON community_deletion_requests +FOR EACH ROW +EXECUTE FUNCTION prevent_community_deletion_request_retargeting(); + +CREATE FUNCTION prevent_community_deletion_approval_removal() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE EXCEPTION 'community deletion approval evidence is immutable' + USING ERRCODE = 'integrity_constraint_violation'; +END; +$$; + +CREATE TRIGGER community_deletion_approval_removal_guard +BEFORE UPDATE OR DELETE ON community_deletion_approvals +FOR EACH ROW +EXECUTE FUNCTION prevent_community_deletion_approval_removal(); + CREATE TABLE community_deletion_checkpoints ( request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, sequence BIGINT GENERATED ALWAYS AS IDENTITY, From cc43b7babb8032385144101a73b0e95d7718b324 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 16:14:32 -0700 Subject: [PATCH 14/51] test: derive owner-limit fixtures from production cap Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/lib.rs | 8 +++++--- crates/buzz-db/src/relay_members.rs | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 92a2ac942b..a24bb838ef 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -6187,8 +6187,10 @@ mod tests { let db = setup_db().await; let owner = format!("{:064x}", Uuid::new_v4().as_u128()); - // Create 3 communities for this owner (the max). - for i in 0..3 { + // Create the maximum number of communities for this owner. Seed from + // the same limit the production path enforces so this remains correct + // when the operator default changes. + for i in 0..relay_members::max_communities_per_owner() { let host = format!("limit-test-{}-{}.example", i, Uuid::new_v4().simple()); assert!(matches!( db.create_community_with_owner(&host, &owner) @@ -6198,7 +6200,7 @@ mod tests { )); } - let host = format!("limit-test-3-{}.example", Uuid::new_v4().simple()); + let host = format!("limit-test-over-{}.example", Uuid::new_v4().simple()); assert_eq!( db.create_community_with_owner(&host, &owner) .await diff --git a/crates/buzz-db/src/relay_members.rs b/crates/buzz-db/src/relay_members.rs index 402229cdec..0dda664eeb 100644 --- a/crates/buzz-db/src/relay_members.rs +++ b/crates/buzz-db/src/relay_members.rs @@ -998,8 +998,10 @@ mod tests { let owner = test_pubkey(); let transferee = test_pubkey(); - // Give the transferee 3 communities (the max). - for _ in 0..3 { + // Give the transferee the maximum number of communities. Seed from + // the same limit the production path enforces so this remains correct + // when the operator default changes. + for _ in 0..max_communities_per_owner() { let c = make_test_community(&pool).await; bootstrap_owner(&pool, c, &transferee) .await From f762fa20f11173d33c3f74d4eb750b3ea38ab63e Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 17:25:29 -0700 Subject: [PATCH 15/51] chore: keep deletion regressions out of github actions Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- .github/workflows/ci.yml | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b3a4d37eac..e65157705a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -353,7 +353,6 @@ jobs: cargo nextest archive \ --cargo-profile ci \ -p buzz-db \ - -p buzz-deletion \ -p buzz-relay \ -p buzz-test-client \ --lib \ @@ -685,20 +684,6 @@ jobs: done cat /tmp/buzz-relay.log exit 1 - - name: Community deletion production-path regressions - run: | - cargo nextest run \ - --archive-file target/ci/backend-integration-tests.tar.zst \ - -E '(package(buzz-deletion) and test(/tests::/)) or (package(buzz-db) and test(/deletion::postgres_tests/)) or (package(buzz-relay) and test(/finalize_push_(holds_serving_lease_through_post_cas_publication|db_failure_after_cas_is_not_success_and_releases_lease)/))' \ - --run-ignored ignored-only - env: - DATABASE_URL: postgres://buzz:${{ env.BUZZ_TEST_POSTGRES_PASSWORD }}@localhost:5432/buzz - BUZZ_TEST_S3_ENDPOINT: http://localhost:9000 - BUZZ_TEST_S3_ACCESS_KEY: buzz_dev - BUZZ_TEST_S3_SECRET_KEY: buzz_dev_secret - BUZZ_TEST_S3_BUCKET: buzz-media - BUZZ_TEST_S3_REGION: us-east-1 - - name: Invite security tests run: | cargo nextest run \ From 4b58b02b486c720e5b992a8d8ff0026ac881174d Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Sun, 2 Aug 2026 18:09:55 -0700 Subject: [PATCH 16/51] fix: freeze destructive deletion evidence Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 34 +++++++++++++++++++++++--- crates/buzz-deletion/src/lib.rs | 12 +++++++++ migrations/0028_community_deletion.sql | 7 ++++++ schema/schema.sql | 7 ++++++ 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index bc82a03b95..5645a44e70 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -872,11 +872,17 @@ impl DeletionStore { let row = sqlx::query( "UPDATE community_deletion_requests \ SET stage = 'approved', updated_at = now(), next_attempt_at = now() \ - WHERE id = $1 RETURNING *", + WHERE id = $1 AND stage = 'inventoried' AND blocked_at IS NULL \ + RETURNING *", ) .bind(request_id) - .fetch_one(&mut *tx) - .await?; + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} changed before approval could be recorded" + )) + })?; tx.commit().await?; row_to_request(row) } @@ -2883,6 +2889,28 @@ mod postgres_tests { .await, Err(DbError::DeletionSafety(_)) )); + for mutation in [ + sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_manifest = '{}'::jsonb WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + sqlx::query( + "UPDATE community_deletion_requests \ + SET destructive_storage_frozen_at = destructive_storage_frozen_at + interval '1 second' \ + WHERE id = $1", + ) + .bind(request.id) + .execute(&db.pool) + .await, + ] { + assert!( + mutation.is_err(), + "frozen destructive storage evidence must be immutable" + ); + } store.mark_drained(&token).await.expect("drain"); store .mark_bindings_removed(&token, serde_json::json!({"keys": 0})) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index f4b2f64d82..a3387c9c52 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -983,6 +983,18 @@ async fn execute_stage( ) -> Result<()> { let request = &claim.request; let token = token_with_current_fence(&claim.lease, request); + if matches!( + request.stage, + DeletionStage::Approved + | DeletionStage::Fenced + | DeletionStage::Drained + | DeletionStage::BindingsRemoved + | DeletionStage::PostgresPurged + | DeletionStage::CachePurged + | DeletionStage::LogicallyVerified + ) { + validate_frozen_inventory(request)?; + } match request.stage { DeletionStage::Approved => { // Approval binds immutable catalog + key taxonomy. Live row counts diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 3b909ac5be..3fc6d28019 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -97,6 +97,13 @@ BEGIN RAISE EXCEPTION 'frozen community deletion inventory is immutable' USING ERRCODE = 'integrity_constraint_violation'; END IF; + IF OLD.destructive_storage_frozen_at IS NOT NULL AND ( + NEW.destructive_storage_manifest IS DISTINCT FROM OLD.destructive_storage_manifest + OR NEW.destructive_storage_frozen_at IS DISTINCT FROM OLD.destructive_storage_frozen_at + ) THEN + RAISE EXCEPTION 'frozen destructive storage manifest is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; RETURN NEW; END; $$; diff --git a/schema/schema.sql b/schema/schema.sql index 27bdea1e78..0e24d57a77 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1162,6 +1162,13 @@ BEGIN RAISE EXCEPTION 'frozen community deletion inventory is immutable' USING ERRCODE = 'integrity_constraint_violation'; END IF; + IF OLD.destructive_storage_frozen_at IS NOT NULL AND ( + NEW.destructive_storage_manifest IS DISTINCT FROM OLD.destructive_storage_manifest + OR NEW.destructive_storage_frozen_at IS DISTINCT FROM OLD.destructive_storage_frozen_at + ) THEN + RAISE EXCEPTION 'frozen destructive storage manifest is immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; RETURN NEW; END; $$; From 3094b79a55eb1f2990a92da85315732fd79ebdd9 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Mon, 3 Aug 2026 00:50:08 -0700 Subject: [PATCH 17/51] refactor: remove unused deletion retention scaffolding Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 43 ---------------------- crates/buzz-db/src/migration.rs | 1 - crates/buzz-deletion/src/lib.rs | 14 ++----- crates/buzz-media/src/bucket_index.rs | 4 -- crates/buzz-relay/src/api/git/transport.rs | 1 - crates/buzz-relay/src/handlers/ingest.rs | 1 - migrations/0028_community_deletion.sql | 15 +------- schema/schema.sql | 14 +------ 8 files changed, 5 insertions(+), 88 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 5645a44e70..30c629ae88 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -37,7 +37,6 @@ pub const CONTROL_PLANE_TABLES: &[&str] = &[ "community_deletion_checkpoints", "community_deletion_executor_heartbeats", "community_deletion_requests", - "community_deletion_retention_exceptions", "community_serving_write_leases", ]; @@ -292,9 +291,6 @@ pub struct StorageManifest { pub media_sidecar_keys: Vec, /// Media upload-record keys among tenant keys. pub media_upload_keys: Vec, - /// Reserved for future retention exceptions. Empty in V1: shared CAS is - /// fleet-wide and intentionally outside per-community inventory. - pub retained_shared_cas_keys: Vec, /// Fleet keys unknown to the current taxonomy. Non-empty fails inventory. pub unknown_keys: Vec, /// Keys whose current object version cannot be safely removed. Non-empty fails. @@ -360,8 +356,6 @@ pub struct DeletionInspection { pub approval: Option, /// Unit checkpoints. pub checkpoints: Vec, - /// Retention exceptions/holds. - pub retention_exceptions: Vec, } /// Explicit approval evidence. @@ -377,21 +371,6 @@ pub struct DeletionApproval { pub approved_at: DateTime, } -/// Physical-expiry retention exception. -#[derive(Debug, Clone, Serialize)] -pub struct RetentionException { - /// Stable hold/exception key. - pub exception_key: String, - /// Reason it survives logical deletion. - pub reason: String, - /// Optional expiry. - pub expires_at: Option>, - /// Creating operator. - pub created_by: String, - /// Creation timestamp. - pub created_at: DateTime, -} - /// Monotonic lease token required by every execution mutation. #[derive(Debug, Clone, PartialEq, Eq)] pub struct LeaseToken { @@ -586,30 +565,10 @@ impl DeletionStore { }) }) .collect::>>()?; - let retention_exceptions = sqlx::query( - "SELECT exception_key, reason, expires_at, created_by, created_at \ - FROM community_deletion_retention_exceptions WHERE request_id = $1 \ - ORDER BY exception_key", - ) - .bind(request_id) - .fetch_all(&self.pool) - .await? - .into_iter() - .map(|row| { - Ok(RetentionException { - exception_key: row.try_get("exception_key")?, - reason: row.try_get("reason")?, - expires_at: row.try_get("expires_at")?, - created_by: row.try_get("created_by")?, - created_at: row.try_get("created_at")?, - }) - }) - .collect::>>()?; Ok(DeletionInspection { request, approval, checkpoints, - retention_exceptions, }) } @@ -2263,7 +2222,6 @@ mod tests { git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), - retained_shared_cas_keys: Vec::new(), unknown_keys: Vec::new(), unsupported_version_keys: Vec::new(), } @@ -2439,7 +2397,6 @@ mod postgres_tests { git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), - retained_shared_cas_keys: Vec::new(), unknown_keys: Vec::new(), unsupported_version_keys: Vec::new(), }, diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 2f340e0715..1fb4a02fac 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -351,7 +351,6 @@ mod tests { "community_deletion_requests", "community_deletion_approvals", "community_deletion_checkpoints", - "community_deletion_retention_exceptions", "community_serving_write_leases", "community_deletion_executor_heartbeats", ] { diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index a3387c9c52..e99d9f6cee 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -593,10 +593,7 @@ fn validate_storage_ownership(request: &DeletionRequest, manifest: &StorageManif .iter() .map(|object| (object.key.clone(), object.size)), ); - if classified.tenant_keys() != manifest.tenant_keys - || !classified.unknown_keys.is_empty() - || !classified.retained_shared_cas_keys.is_empty() - { + if classified.tenant_keys() != manifest.tenant_keys || !classified.unknown_keys.is_empty() { return Err(permanent( "storage manifest contains keys not owned by the deletion target", )); @@ -665,7 +662,6 @@ async fn build_storage_manifest_from_objects( git_pointer_keys: bucket.git_pointer_keys, media_sidecar_keys: bucket.media_sidecar_keys, media_upload_keys: bucket.media_upload_keys, - retained_shared_cas_keys: bucket.retained_shared_cas_keys, unknown_keys: bucket.unknown_keys, unsupported_version_keys, }; @@ -1174,13 +1170,12 @@ async fn execute_stage( .await?; } DeletionStage::LogicallyVerified => { - let frozen = validate_frozen_inventory(request)?; + validate_frozen_inventory(request)?; services .store .mark_retention_pending( &token, serde_json::json!({ - "retained_shared_cas_keys": frozen.storage.retained_shared_cas_keys, "policy": "member-erasure and fleet-wide shared-CAS GC are out of V1 scope" }), ) @@ -1456,7 +1451,6 @@ mod tests { git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), - retained_shared_cas_keys: Vec::new(), unknown_keys: Vec::new(), unsupported_version_keys: Vec::new(), }, @@ -1554,7 +1548,7 @@ mod tests { let mut digest_tampered = claim.request.clone(); digest_tampered.inventory_manifest = Some(serde_json::json!({ "schema": {"revision": 0, "migration_version": 0, "scoped_tables": [], "row_counts": {}, "fenced_tables": []}, - "storage": {"version": 2, "tenant_keys": [], "tenant_objects": [], "git_pointer_keys": [], "media_sidecar_keys": [], "media_upload_keys": [], "retained_shared_cas_keys": [], "unknown_keys": [], "unsupported_version_keys": []} + "storage": {"version": 2, "tenant_keys": [], "tenant_objects": [], "git_pointer_keys": [], "media_sidecar_keys": [], "media_upload_keys": [], "unknown_keys": [], "unsupported_version_keys": []} })); assert!(validate_frozen_inventory(&digest_tampered).is_err()); @@ -1571,7 +1565,6 @@ mod tests { git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), - retained_shared_cas_keys: Vec::new(), unknown_keys: Vec::new(), unsupported_version_keys: Vec::new(), }; @@ -1614,7 +1607,6 @@ mod tests { git_pointer_keys: Vec::new(), media_sidecar_keys: vec![object_key.clone()], media_upload_keys: Vec::new(), - retained_shared_cas_keys: Vec::new(), unknown_keys: Vec::new(), unsupported_version_keys: Vec::new(), }; diff --git a/crates/buzz-media/src/bucket_index.rs b/crates/buzz-media/src/bucket_index.rs index 08dd97f50f..d05656cba2 100644 --- a/crates/buzz-media/src/bucket_index.rs +++ b/crates/buzz-media/src/bucket_index.rs @@ -797,7 +797,6 @@ pub fn deletion_inventory( inventory.media_sidecar_keys.sort(); inventory.media_upload_keys.sort(); inventory.git_pointer_keys.sort(); - inventory.retained_shared_cas_keys.sort(); inventory.unknown_keys.sort(); inventory } @@ -845,8 +844,6 @@ pub struct DeletionBucketInventory { pub media_upload_keys: Vec, /// Community Git repository pointers. pub git_pointer_keys: Vec, - /// Shared immutable media/Git CAS data retained in V1. - pub retained_shared_cas_keys: Vec, /// Bucket keys outside the exact writer taxonomy. pub unknown_keys: Vec, } @@ -886,7 +883,6 @@ mod deletion_inventory_tests { assert_eq!(inventory.media_sidecar_keys.len(), 1); assert_eq!(inventory.media_upload_keys.len(), 1); assert_eq!(inventory.git_pointer_keys.len(), 1); - assert!(inventory.retained_shared_cas_keys.is_empty()); assert!(inventory.unknown_keys.is_empty()); assert_eq!(inventory.tenant_keys().len(), 3); } diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index fc0b774bfa..f662e5b748 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -2345,7 +2345,6 @@ mod track_c_tests { git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), - retained_shared_cas_keys: Vec::new(), unknown_keys: Vec::new(), unsupported_version_keys: Vec::new(), }, diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index 3df2f4b42b..93caa6c0ae 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -3206,7 +3206,6 @@ mod tests { git_pointer_keys: Vec::new(), media_sidecar_keys: Vec::new(), media_upload_keys: Vec::new(), - retained_shared_cas_keys: Vec::new(), unknown_keys: Vec::new(), unsupported_version_keys: Vec::new(), }, diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 3fc6d28019..81dc75ed37 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -146,16 +146,6 @@ CREATE TABLE community_deletion_checkpoints ( CHECK ((status = 'failed') = (error IS NOT NULL)) ); -CREATE TABLE community_deletion_retention_exceptions ( - request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, - exception_key TEXT NOT NULL, - reason TEXT NOT NULL, - expires_at TIMESTAMPTZ, - created_by TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (request_id, exception_key) -); - CREATE TABLE community_serving_write_leases ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), community_id UUID NOT NULL REFERENCES communities(id), @@ -166,8 +156,7 @@ CREATE TABLE community_serving_write_leases ( fence_generation BIGINT NOT NULL CHECK (fence_generation >= 0), lease_until TIMESTAMPTZ NOT NULL, heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - UNIQUE (community_id, id) + created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); CREATE INDEX community_serving_write_leases_active ON community_serving_write_leases (community_id, lease_until); @@ -186,7 +175,6 @@ INSERT INTO _operator_global_tables (table_name, reason) VALUES ('community_deletion_requests', 'deployment deletion lifecycle and frozen inventory'), ('community_deletion_approvals', 'deployment operator destructive approvals'), ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), - ('community_deletion_retention_exceptions', 'deployment retention holds and expiry exceptions'), ('community_serving_write_leases', 'deployment serving side-effect leases drained by deletion'), ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); @@ -208,7 +196,6 @@ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ 'community_deletion_requests', 'community_deletion_approvals', 'community_deletion_checkpoints', - 'community_deletion_retention_exceptions', 'community_serving_write_leases', 'community_deletion_executor_heartbeats' ]::TEXT[]) diff --git a/schema/schema.sql b/schema/schema.sql index 0e24d57a77..5cca2ac9ec 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1210,15 +1210,6 @@ CREATE TABLE community_deletion_checkpoints ( CHECK ((status = 'completed') = (completed_at IS NOT NULL)), CHECK ((status = 'failed') = (error IS NOT NULL)) ); -CREATE TABLE community_deletion_retention_exceptions ( - request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE RESTRICT, - exception_key TEXT NOT NULL, - reason TEXT NOT NULL, - expires_at TIMESTAMPTZ, - created_by TEXT NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (request_id, exception_key) -); CREATE TABLE community_serving_write_leases ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), community_id UUID NOT NULL REFERENCES communities(id), @@ -1229,8 +1220,7 @@ CREATE TABLE community_serving_write_leases ( fence_generation BIGINT NOT NULL CHECK (fence_generation >= 0), lease_until TIMESTAMPTZ NOT NULL, heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT now(), - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - UNIQUE (community_id, id) + created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); CREATE INDEX community_serving_write_leases_active ON community_serving_write_leases (community_id, lease_until); @@ -1248,7 +1238,6 @@ INSERT INTO _operator_global_tables (table_name, reason) VALUES ('community_deletion_requests', 'deployment deletion lifecycle and frozen inventory'), ('community_deletion_approvals', 'deployment operator destructive approvals'), ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), - ('community_deletion_retention_exceptions', 'deployment retention holds and expiry exceptions'), ('community_serving_write_leases', 'deployment serving side-effect leases drained by deletion'), ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); @@ -1265,7 +1254,6 @@ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ 'community_deletion_requests', 'community_deletion_approvals', 'community_deletion_checkpoints', - 'community_deletion_retention_exceptions', 'community_serving_write_leases', 'community_deletion_executor_heartbeats' ]::TEXT[]) From 9415fd81cd3e853db0813963fd7e68b0e3ac18a2 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Mon, 3 Aug 2026 08:26:19 -0700 Subject: [PATCH 18/51] refactor: defer deletion worker helm packaging Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- deploy/charts/buzz/README.md | 64 +------------ .../templates/deletion-worker-deployment.yaml | 94 ------------------- .../buzz/tests/deletion_worker_test.yaml | 52 ---------- deploy/charts/buzz/tests/render_test.yaml | 1 - deploy/charts/buzz/values.schema.json | 18 ---- deploy/charts/buzz/values.yaml | 25 ----- 6 files changed, 1 insertion(+), 253 deletions(-) delete mode 100644 deploy/charts/buzz/templates/deletion-worker-deployment.yaml delete mode 100644 deploy/charts/buzz/tests/deletion_worker_test.yaml diff --git a/deploy/charts/buzz/README.md b/deploy/charts/buzz/README.md index 9ec31282ff..b2778df28b 100644 --- a/deploy/charts/buzz/README.md +++ b/deploy/charts/buzz/README.md @@ -145,68 +145,6 @@ volumes, and each init container must define an appropriate security context and resources. Empty `relay.command` and `relay.args` arrays preserve the image defaults; non-empty values override its entrypoint and arguments respectively. -## Whole-community deletion worker - -Whole-community logical deletion is CLI-only. The default chart remains inert: -`deletionWorker.enabled=false` renders no deletion pod, Service, Ingress, or -route, and the image ENTRYPOINT remains `buzz-relay`. - -Manual OSS operators run the durable engine from the existing image context: - -```sh -buzz-admin deletions submit --host community.example --requested-by operator -buzz-admin deletions approve --approved-by operator -buzz-admin deletions run # one job -buzz-admin deletions drain # current runnable queue -``` - -Hosted operators can opt into a dedicated no-ingress worker: - -```yaml -deletionWorker: - enabled: true - serviceAccountName: buzz-deletion-worker - existingSecret: buzz-deletion-worker-credentials -``` - -The dedicated Secret and service account should grant only the PostgreSQL, -Redis, and S3 delete/list capabilities needed by the worker. Its command is -`buzz-admin deletions worker`; the private `/_liveness` and `/_readiness` -listener exists only for pod probes, and SIGTERM marks readiness draining, -finishes/releases the current durable unit, then exits inside the configured -90-second window. Do not route public traffic to this pod. - -### Required deletion rollout order - -Migration 0027 installs database triggers, but replicas built before this -release do not acquire the external-effect leases that cover S3, Redis, push, -and workflow work. **Do not approve or run a deletion until every serving -replica is this release or newer.** Use this order: - -1. Disable/scale the deletion worker to zero and leave requests unapproved. -2. Apply migration 0027 during a low-traffic window. It sets a 5-second - `lock_timeout`; if any trigger's `SHARE ROW EXCLUSIVE` lock cannot be acquired, - the transaction fails with no partial catalog and should be retried after - clearing/finishing long transactions. -3. Roll every relay replica and verify all old ReplicaSets are at zero plus - `/_readiness` is healthy. Serving readiness accepts migration 27 and later; - destructive claims intentionally require exact migration 27/catalog equality. -4. Verify `buzz_deletion_serving_leases_active` responds to test traffic and - `buzz_deletion_serving_leases_expired` stays near zero. Alert on growing - `buzz_deletion_serving_leases_dead_tuples` and inspect autovacuum if it grows. -5. Enable the worker and only then approve/run requests. - -PostgreSQL trigger attachment and lease-table maintenance add write cost. -Track application p99 for events/media/Git/workflows alongside the lease gauges; -investigate rising dead tuples or expired rows before raising worker scale. - -Object removal is checkpointed per key and each attempt deletes at most -`deletionWorker.storageDeleteBatchSize` (default 100). Deletion is resumable -from durable checkpoints. Missing keys are recorded as `already_missing` and -processing continues, including the crash window where a DELETE committed -before its first PostgreSQL checkpoint. Changed/versioned keys fail closed. -Final verification performs a fresh complete target inventory before advancing. - ## Device pairing relay The chart can run Buzz's stateless pairing WebSocket relay as an independent @@ -261,7 +199,7 @@ default so long-lived WebSocket connections have time to drain. Schema migrations are embedded in the relay binary via `sqlx::migrate!` and run at startup, gated by `BUZZ_AUTO_MIGRATE` (default `true`). Multiple replicas race-safely behind a Postgres advisory lock. `helm upgrade` is the entire upgrade procedure. -If you prefer decoupling migrations from serving, set `migrate.autoMigrate=false`. **In that mode the chart does not run migrations for you** — you own running `buzz-admin migrate` (separate Pod / one-shot Job) against the database before every `helm install` / `helm upgrade`. This release's startup/readiness check requires migration 0027's serving-fence objects (while accepting later additive migrations), so 0027 must finish before deploying these relay pods. Other feature-specific schema freshness remains operator-owned. A pre-upgrade Helm Job for this is on the chart roadmap; the values knob `migrate.preUpgradeJob.enabled` is reserved. +If you prefer decoupling migrations from serving, set `migrate.autoMigrate=false`. **In that mode the chart does not run migrations for you** — you own running `buzz-admin migrate` (separate Pod / one-shot Job) against the database before every `helm install` / `helm upgrade`. Readiness probes only verify DB connectivity, not schema freshness, so a pod will appear healthy against an unmigrated schema and fail under load. A pre-upgrade Helm Job for this is on the chart roadmap; the values knob `migrate.preUpgradeJob.enabled` is reserved. ## Backups diff --git a/deploy/charts/buzz/templates/deletion-worker-deployment.yaml b/deploy/charts/buzz/templates/deletion-worker-deployment.yaml deleted file mode 100644 index 826846002d..0000000000 --- a/deploy/charts/buzz/templates/deletion-worker-deployment.yaml +++ /dev/null @@ -1,94 +0,0 @@ -{{- if .Values.deletionWorker.enabled }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "buzz.fullname" . }}-deletion-worker - labels: - {{- include "buzz.labels" . | nindent 4 }} - app.kubernetes.io/component: deletion-worker -spec: - replicas: {{ .Values.deletionWorker.replicaCount }} - strategy: - type: Recreate - selector: - matchLabels: - {{- include "buzz.selectorLabels" . | nindent 6 }} - app.kubernetes.io/component: deletion-worker - template: - metadata: - labels: - {{- include "buzz.selectorLabels" . | nindent 8 }} - app.kubernetes.io/component: deletion-worker - {{- with .Values.deletionWorker.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - spec: - # No Service/Ingress is created: only private pod health probes listen. - serviceAccountName: {{ required "deletionWorker.serviceAccountName is required when deletionWorker.enabled=true; use a dedicated identity" .Values.deletionWorker.serviceAccountName }} - automountServiceAccountToken: {{ .Values.deletionWorker.automountServiceAccountToken }} - terminationGracePeriodSeconds: {{ .Values.deletionWorker.terminationGracePeriodSeconds }} - securityContext: - {{- toYaml .Values.relay.securityContext | nindent 8 }} - {{- with .Values.image.pullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - containers: - - name: deletion-worker - image: {{ include "buzz.image" . }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - securityContext: - {{- toYaml .Values.relay.containerSecurityContext | nindent 12 }} - command: ["/usr/local/bin/buzz-admin"] - args: ["deletions", "worker"] - ports: - - { name: health, containerPort: {{ .Values.deletionWorker.healthPort }}, protocol: TCP } - env: - - { name: RELAY_URL, value: {{ .Values.relayUrl | quote }} } - - { name: BUZZ_DELETION_HEALTH_ADDR, value: {{ printf "0.0.0.0:%v" .Values.deletionWorker.healthPort | quote }} } - - { name: BUZZ_DELETION_STORAGE_MAX_OBJECTS, value: {{ .Values.deletionWorker.storageMaxObjects | quote }} } - - { name: BUZZ_DELETION_STORAGE_DELETE_BATCH_SIZE, value: {{ .Values.deletionWorker.storageDeleteBatchSize | quote }} } - {{- $s3Endpoint := include "buzz.s3Endpoint" . }} - {{- if $s3Endpoint }} - - { name: BUZZ_S3_ENDPOINT, value: {{ $s3Endpoint | quote }} } - {{- end }} - - { name: BUZZ_S3_BUCKET, value: {{ .Values.s3.bucket | quote }} } - {{- if .Values.s3.region }} - - { name: BUZZ_S3_REGION, value: {{ .Values.s3.region | quote }} } - {{- end }} - - { name: BUZZ_S3_ADDRESSING_STYLE, value: {{ .Values.s3.addressingStyle | quote }} } - - name: DATABASE_URL - valueFrom: - secretKeyRef: - name: {{ required "deletionWorker.existingSecret is required when deletionWorker.enabled=true" .Values.deletionWorker.existingSecret }} - key: DATABASE_URL - - name: REDIS_URL - valueFrom: - secretKeyRef: - name: {{ .Values.deletionWorker.existingSecret }} - key: REDIS_URL - - name: BUZZ_S3_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.deletionWorker.existingSecret }} - key: BUZZ_S3_ACCESS_KEY - - name: BUZZ_S3_SECRET_KEY - valueFrom: - secretKeyRef: - name: {{ .Values.deletionWorker.existingSecret }} - key: BUZZ_S3_SECRET_KEY - {{- with .Values.deletionWorker.extraEnv }} - {{- toYaml . | nindent 12 }} - {{- end }} - livenessProbe: - httpGet: { path: /_liveness, port: health } - periodSeconds: 10 - timeoutSeconds: 3 - readinessProbe: - httpGet: { path: /_readiness, port: health } - periodSeconds: 5 - timeoutSeconds: 3 - resources: - {{- toYaml .Values.deletionWorker.resources | nindent 12 }} -{{- end }} diff --git a/deploy/charts/buzz/tests/deletion_worker_test.yaml b/deploy/charts/buzz/tests/deletion_worker_test.yaml deleted file mode 100644 index a664cb2ba6..0000000000 --- a/deploy/charts/buzz/tests/deletion_worker_test.yaml +++ /dev/null @@ -1,52 +0,0 @@ -suite: deletion worker render -templates: - - templates/deletion-worker-deployment.yaml -tests: - - it: remains completely inert by default - set: - relayUrl: wss://buzz.example.com - ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" - externalPostgresql.url: postgres://u:p@h:5432/d - externalRedis.url: redis://h:6379 - s3.endpoint: http://minio:9000 - s3.accessKey: a - s3.secretKey: s - asserts: - - hasDocuments: - count: 0 - - - it: renders a dedicated no-ingress worker identity only when enabled - set: - relayUrl: wss://buzz.example.com - ownerPubkey: "0000000000000000000000000000000000000000000000000000000000000000" - externalPostgresql.url: postgres://u:p@h:5432/d - externalRedis.url: redis://h:6379 - s3.endpoint: http://minio:9000 - s3.accessKey: a - s3.secretKey: s - deletionWorker.enabled: true - deletionWorker.serviceAccountName: buzz-deletion-worker - deletionWorker.existingSecret: buzz-deletion-worker-credentials - asserts: - - equal: - path: kind - value: Deployment - - equal: - path: spec.template.spec.serviceAccountName - value: buzz-deletion-worker - - equal: - path: spec.template.spec.containers[0].command[0] - value: /usr/local/bin/buzz-admin - - equal: - path: spec.template.spec.containers[0].args - value: [deletions, worker] - - equal: - path: spec.template.spec.containers[0].readinessProbe.httpGet.path - value: /_readiness - - equal: - path: spec.template.spec.containers[0].env[3].value - value: "100" - - notExists: - path: spec.template.spec.containers[0].env[9].valueFrom.secretKeyRef.optional - - notExists: - path: spec.template.spec.containers[0].env[10].valueFrom.secretKeyRef.optional diff --git a/deploy/charts/buzz/tests/render_test.yaml b/deploy/charts/buzz/tests/render_test.yaml index 805d40df87..cf08210781 100644 --- a/deploy/charts/buzz/tests/render_test.yaml +++ b/deploy/charts/buzz/tests/render_test.yaml @@ -3,7 +3,6 @@ suite: production render # (e.g. checksum/secret include of secret-chart.yaml) resolve at render time. templates: - templates/deployment.yaml - - templates/deletion-worker-deployment.yaml - templates/secret-chart.yaml - templates/serviceaccount.yaml - templates/service.yaml diff --git a/deploy/charts/buzz/values.schema.json b/deploy/charts/buzz/values.schema.json index 2777fea512..e1e362a531 100644 --- a/deploy/charts/buzz/values.schema.json +++ b/deploy/charts/buzz/values.schema.json @@ -270,24 +270,6 @@ "labels": { "type": "object" } } }, - "deletionWorker": { - "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { "type": "boolean" }, - "replicaCount": { "type": "integer", "minimum": 1 }, - "serviceAccountName": { "type": "string" }, - "automountServiceAccountToken": { "type": "boolean" }, - "existingSecret": { "type": "string" }, - "healthPort": { "type": "integer", "minimum": 1, "maximum": 65535 }, - "terminationGracePeriodSeconds": { "type": "integer", "minimum": 1 }, - "storageMaxObjects": { "type": "integer", "minimum": 1 }, - "storageDeleteBatchSize": { "type": "integer", "minimum": 1, "maximum": 10000 }, - "podAnnotations": { "type": "object" }, - "extraEnv": { "type": "array", "items": { "type": "object" } }, - "resources": { "type": "object" } - } - }, "pairingRelay": { "type": "object", "additionalProperties": false, diff --git a/deploy/charts/buzz/values.yaml b/deploy/charts/buzz/values.yaml index cced4ecca9..42b09f1b3e 100644 --- a/deploy/charts/buzz/values.yaml +++ b/deploy/charts/buzz/values.yaml @@ -397,28 +397,3 @@ serviceMonitor: # ── Free-form extra manifests ──────────────────────────────────────────────── extraManifests: [] - -# ── Whole-community deletion worker ────────────────────────────────────────── -# Strictly inert by default. Hosted operators may enable this dedicated, -# no-ingress identity; manual OSS operation uses `buzz-admin deletions run|drain` -# from the existing image instead. -deletionWorker: - enabled: false - replicaCount: 1 - serviceAccountName: "" - automountServiceAccountToken: false - existingSecret: "" - healthPort: 8080 - terminationGracePeriodSeconds: 90 - storageMaxObjects: 1000000 - # Durable per-key checkpoints bound one worker attempt to this many deletes. - storageDeleteBatchSize: 100 - podAnnotations: {} - extraEnv: [] - resources: - requests: - cpu: "250m" - memory: "256Mi" - limits: - cpu: "1" - memory: "1Gi" From d2376be05b5e03d8e2c907a28ac747488e7aae1a Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Mon, 3 Aug 2026 09:05:33 -0700 Subject: [PATCH 19/51] fix(deletion): support desired-state schema deployments Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 62 ++++++++++++------- migrations/0028_community_deletion.sql | 41 ++++++++++++- schema/schema.sql | 83 +++++++++++++++++++++++++- scripts/attach-schema-partitions.sql | 14 +++++ 4 files changed, 171 insertions(+), 29 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 30c629ae88..5f5036d189 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -445,14 +445,17 @@ impl DeletionStore { } /// Check deletion control-plane/schema connectivity. + /// + /// Probe the deployed catalog rather than SQLx's migration ledger. Buzz also + /// supports desired-state schema application through `pgschema`, which creates + /// the same deletion objects without creating `_sqlx_migrations`. pub async fn ping(&self) -> bool { - sqlx::query_scalar::<_, i32>( - "SELECT 1 FROM _sqlx_migrations WHERE version = $1 AND success LIMIT 1", + sqlx::query_scalar::<_, bool>( + "SELECT to_regclass('community_deletion_requests') IS NOT NULL", ) - .bind(EXPECTED_MIGRATION_VERSION) - .fetch_optional(&self.pool) + .fetch_one(&self.pool) .await - .is_ok_and(|row| row == Some(1)) + .unwrap_or(false) } /// Persist a request. Only active non-tombstone communities may be submitted. @@ -578,8 +581,7 @@ impl DeletionStore { /// and rollback. Destructive inventory still calls [`Self::validate_catalog`] /// and requires exact migration/table equality. pub async fn validate_serving_catalog(&self) -> Result<()> { - let migration_version = self.live_migration_version().await?; - validate_serving_migration_version(migration_version)?; + validate_serving_migration_version(self.live_migration_version().await?)?; let runtime_columns = sqlx::query( "SELECT attname, format_type(atttypid, atttypmod) AS type_name, attnotnull \ @@ -691,8 +693,7 @@ impl DeletionStore { /// Unlike relay serving compatibility, this intentionally rejects newer /// migrations and unknown tenant tables until the deletion manifest changes. pub async fn validate_catalog(&self) -> Result<()> { - let migration_version = self.live_migration_version().await?; - validate_destructive_migration_version(migration_version)?; + validate_destructive_migration_version(self.live_migration_version().await?)?; let expected = EXPECTED_SCOPED_TABLES .iter() @@ -1832,8 +1833,17 @@ impl DeletionStore { Ok(()) } - async fn live_migration_version(&self) -> Result { - sqlx::query_scalar("SELECT COALESCE(max(version), 0) FROM _sqlx_migrations WHERE success") + async fn live_migration_version(&self) -> Result> { + let ledger_exists: bool = + sqlx::query_scalar("SELECT to_regclass('_sqlx_migrations') IS NOT NULL") + .fetch_one(&self.pool) + .await?; + if !ledger_exists { + // Desired-state `pgschema` applies the checked-in final catalog but + // intentionally does not synthesize SQLx migration history. + return Ok(None); + } + sqlx::query_scalar("SELECT max(version) FROM _sqlx_migrations WHERE success") .fetch_one(&self.pool) .await .map_err(Into::into) @@ -1887,20 +1897,22 @@ impl DeletionStore { } } -fn validate_destructive_migration_version(migration_version: i64) -> Result<()> { - if migration_version != EXPECTED_MIGRATION_VERSION { +fn validate_destructive_migration_version(migration_version: Option) -> Result<()> { + if migration_version.is_some_and(|version| version != EXPECTED_MIGRATION_VERSION) { Err(DbError::DeletionSafety(format!( - "community deletion schema migration drift: expected {EXPECTED_MIGRATION_VERSION}, got {migration_version}" + "community deletion schema migration drift: expected {EXPECTED_MIGRATION_VERSION}, got {}", + migration_version.expect("checked Some") ))) } else { Ok(()) } } -fn validate_serving_migration_version(migration_version: i64) -> Result<()> { - if migration_version < EXPECTED_MIGRATION_VERSION { +fn validate_serving_migration_version(migration_version: Option) -> Result<()> { + if migration_version.is_some_and(|version| version < EXPECTED_MIGRATION_VERSION) { Err(DbError::DeletionSafety(format!( - "community serving fence migration is too old: require at least {EXPECTED_MIGRATION_VERSION}, got {migration_version}" + "community serving fence migration is too old: require at least {EXPECTED_MIGRATION_VERSION}, got {}", + migration_version.expect("checked Some") ))) } else { Ok(()) @@ -2272,12 +2284,16 @@ mod tests { } #[test] - fn serving_catalog_accepts_future_migrations_but_destructive_catalog_does_not() { - assert!(validate_serving_migration_version(EXPECTED_MIGRATION_VERSION).is_ok()); - assert!(validate_serving_migration_version(EXPECTED_MIGRATION_VERSION + 1).is_ok()); - assert!(validate_serving_migration_version(EXPECTED_MIGRATION_VERSION - 1).is_err()); - assert!(validate_destructive_migration_version(EXPECTED_MIGRATION_VERSION).is_ok()); - assert!(validate_destructive_migration_version(EXPECTED_MIGRATION_VERSION + 1).is_err()); + fn migration_checks_accept_desired_state_catalogs_without_sqlx_ledger() { + assert!(validate_serving_migration_version(None).is_ok()); + assert!(validate_destructive_migration_version(None).is_ok()); + assert!(validate_serving_migration_version(Some(EXPECTED_MIGRATION_VERSION)).is_ok()); + assert!(validate_serving_migration_version(Some(EXPECTED_MIGRATION_VERSION + 1)).is_ok()); + assert!(validate_serving_migration_version(Some(EXPECTED_MIGRATION_VERSION - 1)).is_err()); + assert!(validate_destructive_migration_version(Some(EXPECTED_MIGRATION_VERSION)).is_ok()); + assert!( + validate_destructive_migration_version(Some(EXPECTED_MIGRATION_VERSION + 1)).is_err() + ); } #[test] diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 81dc75ed37..810a8210cb 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -356,11 +356,11 @@ BEGIN FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.oid = target - AND n.nspname = 'public' + AND n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND NOT c.relispartition; IF NOT FOUND THEN - RAISE EXCEPTION 'community write fence target % is not a public table', target + RAISE EXCEPTION 'community write fence target % is not a table in the current schema', target USING ERRCODE = 'wrong_object_type'; END IF; IF community_write_fence_excluded_table(relation_name) THEN @@ -400,7 +400,7 @@ BEGIN FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace JOIN pg_attribute a ON a.attrelid = c.oid - WHERE n.nspname = 'public' + WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND NOT c.relispartition AND a.attname = 'community_id' @@ -412,3 +412,38 @@ BEGIN END LOOP; END $$; + +-- Desired-state schema application does not replay migration history, so keep +-- these explicit calls as first-class catalog declarations. They also make the +-- fence contract visible to migration linting instead of hiding it only in the +-- dynamic bootstrap loop above. +SELECT attach_community_write_fence('api_tokens'); +SELECT attach_community_write_fence('archived_identities'); +SELECT attach_community_write_fence('audit_log'); +SELECT attach_community_write_fence('channel_members'); +SELECT attach_community_write_fence('channels'); +SELECT attach_community_write_fence('community_bans'); +SELECT attach_community_write_fence('delivery_log'); +SELECT attach_community_write_fence('event_mentions'); +SELECT attach_community_write_fence('events'); +SELECT attach_community_write_fence('git_repo_names'); +SELECT attach_community_write_fence('join_policy_acceptances'); +SELECT attach_community_write_fence('moderation_actions'); +SELECT attach_community_write_fence('moderation_reports'); +SELECT attach_community_write_fence('parameterized_event_watermarks'); +SELECT attach_community_write_fence('product_feedback'); +SELECT attach_community_write_fence('pubkey_allowlist'); +SELECT attach_community_write_fence('push_leases'); +SELECT attach_community_write_fence('push_match_queue'); +SELECT attach_community_write_fence('push_wake_outbox'); +SELECT attach_community_write_fence('rate_limit_violations'); +SELECT attach_community_write_fence('reactions'); +SELECT attach_community_write_fence('relay_invites'); +SELECT attach_community_write_fence('relay_members'); +SELECT attach_community_write_fence('scheduled_workflow_fires'); +SELECT attach_community_write_fence('subscriptions'); +SELECT attach_community_write_fence('thread_metadata'); +SELECT attach_community_write_fence('users'); +SELECT attach_community_write_fence('workflow_approvals'); +SELECT attach_community_write_fence('workflow_runs'); +SELECT attach_community_write_fence('workflows'); diff --git a/schema/schema.sql b/schema/schema.sql index 5cca2ac9ec..53b30bb258 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -794,6 +794,48 @@ INSERT INTO _operator_global_tables (table_name, reason) VALUES ('communities', 'the tenant registry itself; id IS the community key'), ('rate_limit_violations', 'deployment abuse/health; never tenant-observable; community_id is an attribution label only'), ('_operator_global_tables', 'the registry table itself'); + +-- ── Additive tenant tables represented in migrations 0002/0007/0017 ────────── +-- Keep desired-state schema parity with the embedded SQLx migration path. +CREATE TABLE git_repo_names ( + community_id UUID NOT NULL REFERENCES communities(id), + repo_id TEXT NOT NULL, + owner_pubkey TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (community_id, repo_id) +); +CREATE INDEX idx_git_repo_names_owner ON git_repo_names (community_id, owner_pubkey); + +CREATE TABLE parameterized_event_watermarks ( + community_id UUID NOT NULL REFERENCES communities(id), + kind INT NOT NULL, + pubkey BYTEA NOT NULL, + d_tag TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + event_id BYTEA NOT NULL, + PRIMARY KEY (community_id, kind, pubkey, d_tag) +); +CREATE INDEX idx_event_mentions_community_event + ON event_mentions (community_id, event_id); + +CREATE TABLE product_feedback ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + community_id UUID NOT NULL REFERENCES communities(id), + event_id BYTEA NOT NULL CHECK (length(event_id) = 32), + submitter_pubkey BYTEA NOT NULL CHECK (length(submitter_pubkey) = 32), + category TEXT CHECK (category IN ('bug', 'praise', 'needs-work')), + body TEXT NOT NULL CHECK (length(btrim(body)) > 0), + tags JSONB NOT NULL DEFAULT '[]'::jsonb CHECK (jsonb_typeof(tags) = 'array'), + event_created_at TIMESTAMPTZ NOT NULL, + received_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (event_id) +); +CREATE INDEX idx_product_feedback_received + ON product_feedback (received_at DESC, id); +CREATE INDEX idx_product_feedback_community_received + ON product_feedback (community_id, received_at DESC, id); +INSERT INTO _operator_global_tables (table_name, reason) VALUES + ('product_feedback', 'deployment product inbox; community_id is provenance only'); -- NIP-PL effective lease state and durable wake outbox. Every key is led by -- community_id: client-provided origin is confirmation only, never routing. CREATE TABLE push_leases ( @@ -1399,11 +1441,11 @@ BEGIN FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.oid = target - AND n.nspname = 'public' + AND n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND NOT c.relispartition; IF NOT FOUND THEN - RAISE EXCEPTION 'community write fence target % is not a public table', target + RAISE EXCEPTION 'community write fence target % is not a table in the current schema', target USING ERRCODE = 'wrong_object_type'; END IF; IF community_write_fence_excluded_table(relation_name) THEN @@ -1443,7 +1485,7 @@ BEGIN FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace JOIN pg_attribute a ON a.attrelid = c.oid - WHERE n.nspname = 'public' + WHERE n.nspname = current_schema() AND c.relkind IN ('r', 'p') AND NOT c.relispartition AND a.attname = 'community_id' @@ -1455,3 +1497,38 @@ BEGIN END LOOP; END $$; + +-- Desired-state schema application does not replay migration history, so keep +-- these explicit calls as first-class catalog declarations. They also make the +-- fence contract visible to migration linting instead of hiding it only in the +-- dynamic bootstrap loop above. +SELECT attach_community_write_fence('api_tokens'); +SELECT attach_community_write_fence('archived_identities'); +SELECT attach_community_write_fence('audit_log'); +SELECT attach_community_write_fence('channel_members'); +SELECT attach_community_write_fence('channels'); +SELECT attach_community_write_fence('community_bans'); +SELECT attach_community_write_fence('delivery_log'); +SELECT attach_community_write_fence('event_mentions'); +SELECT attach_community_write_fence('events'); +SELECT attach_community_write_fence('git_repo_names'); +SELECT attach_community_write_fence('join_policy_acceptances'); +SELECT attach_community_write_fence('moderation_actions'); +SELECT attach_community_write_fence('moderation_reports'); +SELECT attach_community_write_fence('parameterized_event_watermarks'); +SELECT attach_community_write_fence('product_feedback'); +SELECT attach_community_write_fence('pubkey_allowlist'); +SELECT attach_community_write_fence('push_leases'); +SELECT attach_community_write_fence('push_match_queue'); +SELECT attach_community_write_fence('push_wake_outbox'); +SELECT attach_community_write_fence('rate_limit_violations'); +SELECT attach_community_write_fence('reactions'); +SELECT attach_community_write_fence('relay_invites'); +SELECT attach_community_write_fence('relay_members'); +SELECT attach_community_write_fence('scheduled_workflow_fires'); +SELECT attach_community_write_fence('subscriptions'); +SELECT attach_community_write_fence('thread_metadata'); +SELECT attach_community_write_fence('users'); +SELECT attach_community_write_fence('workflow_approvals'); +SELECT attach_community_write_fence('workflow_runs'); +SELECT attach_community_write_fence('workflows'); diff --git a/scripts/attach-schema-partitions.sql b/scripts/attach-schema-partitions.sql index d02fc7c82a..5837676f84 100644 --- a/scripts/attach-schema-partitions.sql +++ b/scripts/attach-schema-partitions.sql @@ -21,6 +21,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_past; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_past; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_past; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p_past; ALTER TABLE events ATTACH PARTITION events_p_past FOR VALUES FROM (MINVALUE) TO ('2026-01-01'); END IF; @@ -33,6 +34,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_01; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_01; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_01; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_01; ALTER TABLE events ATTACH PARTITION events_p2026_01 FOR VALUES FROM ('2026-01-01') TO ('2026-02-01'); END IF; @@ -45,6 +47,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_02; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_02; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_02; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_02; ALTER TABLE events ATTACH PARTITION events_p2026_02 FOR VALUES FROM ('2026-02-01') TO ('2026-03-01'); END IF; @@ -57,6 +60,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_03; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_03; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_03; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_03; ALTER TABLE events ATTACH PARTITION events_p2026_03 FOR VALUES FROM ('2026-03-01') TO ('2026-04-01'); END IF; @@ -69,6 +73,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_04; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_04; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_04; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_04; ALTER TABLE events ATTACH PARTITION events_p2026_04 FOR VALUES FROM ('2026-04-01') TO ('2026-05-01'); END IF; @@ -81,6 +86,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_05; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_05; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_05; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_05; ALTER TABLE events ATTACH PARTITION events_p2026_05 FOR VALUES FROM ('2026-05-01') TO ('2026-06-01'); END IF; @@ -93,6 +99,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p2026_06; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p2026_06; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p2026_06; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p2026_06; ALTER TABLE events ATTACH PARTITION events_p2026_06 FOR VALUES FROM ('2026-06-01') TO ('2026-07-01'); END IF; @@ -105,6 +112,7 @@ BEGIN DROP TRIGGER IF EXISTS events_enqueue_push_match ON events_p_future; DROP TRIGGER IF EXISTS events_refresh_channel_ttl ON events_p_future; DROP TRIGGER IF EXISTS events_created_at_floor ON events_p_future; + DROP TRIGGER IF EXISTS community_write_fence_events ON events_p_future; ALTER TABLE events ATTACH PARTITION events_p_future FOR VALUES FROM ('2026-07-01') TO (MAXVALUE); END IF; @@ -122,6 +130,7 @@ BEGIN AND inhrelid = 'delivery_log_p_past'::regclass ) THEN ALTER TABLE delivery_log_p_past ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p_past; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p_past FOR VALUES FROM (MINVALUE) TO ('2026-03-01'); END IF; @@ -132,6 +141,7 @@ BEGIN AND inhrelid = 'delivery_log_p2026_03'::regclass ) THEN ALTER TABLE delivery_log_p2026_03 ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p2026_03; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p2026_03 FOR VALUES FROM ('2026-03-01') TO ('2026-04-01'); END IF; @@ -142,6 +152,7 @@ BEGIN AND inhrelid = 'delivery_log_p2026_04'::regclass ) THEN ALTER TABLE delivery_log_p2026_04 ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p2026_04; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p2026_04 FOR VALUES FROM ('2026-04-01') TO ('2026-05-01'); END IF; @@ -152,6 +163,7 @@ BEGIN AND inhrelid = 'delivery_log_p2026_05'::regclass ) THEN ALTER TABLE delivery_log_p2026_05 ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p2026_05; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p2026_05 FOR VALUES FROM ('2026-05-01') TO ('2026-06-01'); END IF; @@ -162,6 +174,7 @@ BEGIN AND inhrelid = 'delivery_log_p2026_06'::regclass ) THEN ALTER TABLE delivery_log_p2026_06 ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p2026_06; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p2026_06 FOR VALUES FROM ('2026-06-01') TO ('2026-07-01'); END IF; @@ -172,6 +185,7 @@ BEGIN AND inhrelid = 'delivery_log_p_future'::regclass ) THEN ALTER TABLE delivery_log_p_future ALTER COLUMN id DROP IDENTITY IF EXISTS; + DROP TRIGGER IF EXISTS community_write_fence_delivery_log ON delivery_log_p_future; ALTER TABLE delivery_log ATTACH PARTITION delivery_log_p_future FOR VALUES FROM ('2026-07-01') TO (MAXVALUE); END IF; From 6faede275daab82c9f7eea163bccc22379a8d543 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Mon, 3 Aug 2026 09:36:50 -0700 Subject: [PATCH 20/51] fix: authorize nip-rs rows during community purge Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 5f5036d189..caefd86678 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -1304,6 +1304,12 @@ impl DeletionStore { let mut tx = self.pool.begin().await?; verify_lease_and_fence(&mut tx, token, DeletionStage::BindingsRemoved, generation).await?; set_executor_gucs(&mut tx, token.community_id, generation).await?; + // Migration 0011 fences hard deletion of NIP-RS rows against legacy + // writers. Whole-community deletion is an intentional hard-delete path, + // and the transaction is already bound to an approved, fenced tenant. + sqlx::query("SELECT set_config('buzz.nip_rs_hard_delete', 'on', true)") + .execute(&mut *tx) + .await?; let mut deleted = BTreeMap::new(); // The order is child-before-parent/FK-safe, not alphabetical. Cascades @@ -2829,6 +2835,24 @@ mod postgres_tests { let (db, store) = store().await; let (request, inventory) = inventoried_request(&db, &store).await; let host = request.community_host.clone(); + let read_state_d_tag = format!("read-state:{}", "a".repeat(32)); + sqlx::query( + "INSERT INTO events \ + (community_id, id, pubkey, created_at, kind, tags, content, sig, d_tag) \ + VALUES ($1, $2, $3, now(), 30078, $4, '', $5, $6)", + ) + .bind(request.community_id.as_uuid()) + .bind(vec![1_u8; 32]) + .bind(vec![2_u8; 32]) + .bind(serde_json::json!([ + ["d", &read_state_d_tag], + ["t", "read-state"] + ])) + .bind(vec![3_u8; 64]) + .bind(&read_state_d_tag) + .execute(&db.pool) + .await + .expect("insert guarded NIP-RS row"); store .approve(request.id, "approver", None) .await From bc3b37339830e8a0eb236e59c3a8cf9fadd587c6 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Mon, 3 Aug 2026 10:16:21 -0700 Subject: [PATCH 21/51] refactor: narrow community deletion to operator execution Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- Cargo.lock | 1 - crates/buzz-admin/src/deletions.rs | 6 +- crates/buzz-db/src/lib.rs | 8 +- crates/buzz-db/src/relay_members.rs | 6 +- crates/buzz-deletion/Cargo.toml | 1 - crates/buzz-deletion/src/lib.rs | 192 +----------------- crates/buzz-relay/src/api/mesh_demo.rs | 31 +-- .../buzz-relay/src/handlers/side_effects.rs | 49 ++--- crates/buzz-test-client/tests/e2e_git.rs | 63 +----- .../tests/e2e_media_extended.rs | 25 +-- crates/buzz-test-client/tests/e2e_relay.rs | 114 +++++------ 11 files changed, 103 insertions(+), 393 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95e26b150d..4e05812cc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1025,7 +1025,6 @@ name = "buzz-deletion" version = "0.1.0" dependencies = [ "anyhow", - "axum", "buzz-core", "buzz-db", "buzz-media", diff --git a/crates/buzz-admin/src/deletions.rs b/crates/buzz-admin/src/deletions.rs index 95d07588a2..64cb8bd732 100644 --- a/crates/buzz-admin/src/deletions.rs +++ b/crates/buzz-admin/src/deletions.rs @@ -12,8 +12,8 @@ mod tests { use clap::Parser; #[test] - fn default_command_does_not_start_a_deletion_worker() { - let command = crate::Cli::try_parse_from(["buzz-admin", "list-members"]); - assert!(command.is_ok()); + fn continuous_worker_command_is_not_exposed() { + let command = crate::Cli::try_parse_from(["buzz-admin", "deletions", "worker"]); + assert!(command.is_err()); } } diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index a24bb838ef..92a2ac942b 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -6187,10 +6187,8 @@ mod tests { let db = setup_db().await; let owner = format!("{:064x}", Uuid::new_v4().as_u128()); - // Create the maximum number of communities for this owner. Seed from - // the same limit the production path enforces so this remains correct - // when the operator default changes. - for i in 0..relay_members::max_communities_per_owner() { + // Create 3 communities for this owner (the max). + for i in 0..3 { let host = format!("limit-test-{}-{}.example", i, Uuid::new_v4().simple()); assert!(matches!( db.create_community_with_owner(&host, &owner) @@ -6200,7 +6198,7 @@ mod tests { )); } - let host = format!("limit-test-over-{}.example", Uuid::new_v4().simple()); + let host = format!("limit-test-3-{}.example", Uuid::new_v4().simple()); assert_eq!( db.create_community_with_owner(&host, &owner) .await diff --git a/crates/buzz-db/src/relay_members.rs b/crates/buzz-db/src/relay_members.rs index 0dda664eeb..402229cdec 100644 --- a/crates/buzz-db/src/relay_members.rs +++ b/crates/buzz-db/src/relay_members.rs @@ -998,10 +998,8 @@ mod tests { let owner = test_pubkey(); let transferee = test_pubkey(); - // Give the transferee the maximum number of communities. Seed from - // the same limit the production path enforces so this remains correct - // when the operator default changes. - for _ in 0..max_communities_per_owner() { + // Give the transferee 3 communities (the max). + for _ in 0..3 { let c = make_test_community(&pool).await; bootstrap_owner(&pool, c, &transferee) .await diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml index 9a3593b665..fb94cea13f 100644 --- a/crates/buzz-deletion/Cargo.toml +++ b/crates/buzz-deletion/Cargo.toml @@ -10,7 +10,6 @@ description = "Durable whole-community deletion engine for Buzz" [dependencies] anyhow = { workspace = true } thiserror = { workspace = true } -axum = { workspace = true } buzz-core = { workspace = true } buzz-db = { workspace = true } buzz-media = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index e99d9f6cee..43171c6642 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -2,6 +2,7 @@ #![warn(missing_docs)] //! Shared durable whole-community deletion engine and store adapters. +#[cfg(test)] use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -19,7 +20,6 @@ use tokio_util::sync::CancellationToken; use uuid::Uuid; const DEFAULT_STORAGE_OBJECT_CAP: u64 = 1_000_000; -const WORKER_IDLE_POLL: Duration = Duration::from_secs(5); const RETRY_DELAY: Duration = Duration::from_secs(30); const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10); const STORAGE_DELETE_BATCH_SIZE: usize = 100; @@ -38,10 +38,6 @@ fn heartbeat_interval() -> Duration { HEARTBEAT_INTERVAL } -fn worker_health_stale_after() -> u64 { - HEARTBEAT_INTERVAL.as_secs().saturating_mul(3) -} - #[derive(Debug, Clone, thiserror::Error)] #[error("deletion execution lease heartbeat failed")] struct DeletionLeaseLost; @@ -264,19 +260,12 @@ pub enum Command { #[arg(long)] executor_id: Option, }, - /// Poll continuously in a dedicated no-ingress worker process. - Worker { - /// Executor identity (defaults to hostname/pid). - #[arg(long)] - executor_id: Option, - }, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum LoopMode { Run, Drain, - Worker, } impl LoopMode { @@ -284,7 +273,6 @@ impl LoopMode { match self { Self::Run => "run", Self::Drain => "drain", - Self::Worker => "worker", } } } @@ -333,36 +321,6 @@ fn is_permanent_error(error: &anyhow::Error) -> bool { }) } -#[derive(Default)] -struct WorkerHealth { - draining: AtomicBool, - dependencies_ready: AtomicBool, - last_heartbeat_epoch: AtomicU64, -} - -impl WorkerHealth { - fn mark_heartbeat(&self) { - self.last_heartbeat_epoch.store( - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(), - Ordering::Relaxed, - ); - } - - fn ready(&self) -> bool { - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - !self.draining.load(Ordering::Relaxed) - && self.dependencies_ready.load(Ordering::Relaxed) - && now.saturating_sub(self.last_heartbeat_epoch.load(Ordering::Relaxed)) - <= worker_health_stale_after() - } -} - #[derive(Debug, Serialize)] struct RunOutput { request_id: Uuid, @@ -439,15 +397,6 @@ pub async fn run(command: Command) -> Result { ) .await } - Command::Worker { executor_id } => { - run_loop( - services, - LoopMode::Worker, - None, - executor_id.unwrap_or_else(default_executor_id), - ) - .await - } } } @@ -687,16 +636,6 @@ async fn run_loop( executor_id: String, ) -> Result { let shutdown = shutdown_token(); - let health = Arc::new(WorkerHealth::default()); - health.mark_heartbeat(); - health - .dependencies_ready - .store(dependencies_ready(&services).await, Ordering::Relaxed); - let _health_task = if mode == LoopMode::Worker { - Some(spawn_worker_health(shutdown.clone(), Arc::clone(&health)).await?) - } else { - None - }; let mut ran = false; loop { if shutdown.is_cancelled() { @@ -722,28 +661,15 @@ async fn run_loop( } }; let Some(claim) = claim else { - match mode { - LoopMode::Worker => { - tokio::select! { - _ = shutdown.cancelled() => continue, - _ = tokio::time::sleep(WORKER_IDLE_POLL) => { - health.dependencies_ready.store( - dependencies_ready(&services).await, - Ordering::Relaxed, - ); - health.mark_heartbeat(); - continue; - }, - } - } - LoopMode::Run if !ran => anyhow::bail!( + if mode == LoopMode::Run && !ran { + anyhow::bail!( "deletion request is not runnable, is blocked, or is leased by another executor" - ), - LoopMode::Run | LoopMode::Drain => return Ok(0), + ); } + return Ok(0); }; ran = true; - let output = execute_claim(&services, mode, claim, &shutdown, Arc::clone(&health)).await?; + let output = execute_claim(&services, mode, claim, &shutdown).await?; print_json(&output)?; if mode == LoopMode::Run || shutdown.is_cancelled() { return Ok(i32::from(output.blocked_reason.is_some())); @@ -796,7 +722,6 @@ async fn execute_claim( mode: LoopMode, mut claim: ClaimedDeletion, shutdown: &CancellationToken, - health: Arc, ) -> Result { let token = claim.lease.clone(); loop { @@ -809,10 +734,7 @@ async fn execute_claim( .store .heartbeat(&token, mode.as_str(), DEFAULT_LEASE_DURATION, false) .await?; - health.dependencies_ready.store(true, Ordering::Relaxed); - health.mark_heartbeat(); - let stage_result = - run_stage_with_heartbeat(services, mode, &claim, shutdown, Arc::clone(&health)).await; + let stage_result = run_stage_with_heartbeat(services, mode, &claim, shutdown).await; match stage_result { StageOutcome::Completed => {} StageOutcome::Shutdown => { @@ -846,25 +768,6 @@ async fn execute_claim( } } -async fn dependencies_ready(services: &Services) -> bool { - if !services.store.ping().await { - return false; - } - let redis_ok = match services.redis.get().await { - Ok(mut connection) => tokio::time::timeout( - Duration::from_secs(5), - redis::cmd("PING").query_async::(&mut *connection), - ) - .await - .is_ok_and(|result| result.is_ok()), - Err(_) => false, - }; - let storage_ok = tokio::time::timeout(Duration::from_secs(5), services.media.ping()) - .await - .is_ok_and(|result| result.is_ok()); - redis_ok && storage_ok -} - enum StageOutcome { Completed, Shutdown, @@ -895,14 +798,12 @@ async fn run_stage_with_heartbeat( mode: LoopMode, claim: &ClaimedDeletion, shutdown: &CancellationToken, - health: Arc, ) -> StageOutcome { let heartbeat_services = services.clone(); let heartbeat_token = claim.lease.clone(); let heartbeat_mode = mode.as_str(); let heartbeat_shutdown = CancellationToken::new(); let heartbeat_cancel = heartbeat_shutdown.clone(); - let heartbeat_health = Arc::clone(&health); let heartbeat_error = CancellationToken::new(); let heartbeat_error_signal = heartbeat_error.clone(); let heartbeat = tokio::spawn(async move { @@ -927,7 +828,6 @@ async fn run_stage_with_heartbeat( heartbeat_error_signal.cancel(); return; } - heartbeat_health.mark_heartbeat(); } } } @@ -1336,48 +1236,6 @@ fn default_executor_id() -> String { format!("{hostname}:{}", std::process::id()) } -async fn spawn_worker_health( - shutdown: CancellationToken, - health: Arc, -) -> Result> { - let address = - std::env::var("BUZZ_DELETION_HEALTH_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".to_string()); - let listener = tokio::net::TcpListener::bind(&address) - .await - .with_context(|| format!("bind deletion worker health endpoint {address}"))?; - let signal = Arc::clone(&health); - let cancel = shutdown.clone(); - tokio::spawn(async move { - cancel.cancelled().await; - signal.draining.store(true, Ordering::Relaxed); - }); - let router = axum::Router::new() - .route("/_liveness", axum::routing::get(|| async { "ok" })) - .route( - "/_readiness", - axum::routing::get({ - move || { - let health = Arc::clone(&health); - async move { - if health.ready() { - (axum::http::StatusCode::OK, "ready") - } else { - (axum::http::StatusCode::SERVICE_UNAVAILABLE, "not_ready") - } - } - } - }), - ); - Ok(tokio::spawn(async move { - if let Err(error) = axum::serve(listener, router) - .with_graceful_shutdown(shutdown.cancelled_owned()) - .await - { - eprintln!("deletion worker health endpoint failed: {error}"); - } - })) -} - fn shutdown_token() -> CancellationToken { let token = CancellationToken::new(); let signal = token.clone(); @@ -1696,21 +1554,7 @@ mod tests { } #[test] - fn worker_readiness_requires_dependencies_heartbeat_and_not_draining() { - let health = WorkerHealth::default(); - assert!(!health.ready()); - health.dependencies_ready.store(true, Ordering::Relaxed); - health.mark_heartbeat(); - assert!(health.ready()); - health.draining.store(true, Ordering::Relaxed); - assert!(!health.ready()); - health.draining.store(false, Ordering::Relaxed); - health.last_heartbeat_epoch.store(1, Ordering::Relaxed); - assert!(!health.ready()); - } - - #[test] - fn worker_configuration_requires_every_destructive_dependency() { + fn deletion_configuration_requires_every_destructive_dependency() { let variable = format!("BUZZ_DELETION_REQUIRED_TEST_{}", Uuid::new_v4().simple()); assert!(required_env(&variable).is_err()); std::env::set_var(&variable, " "); @@ -1869,14 +1713,7 @@ mod tests { let cancel = shutdown.clone(); let services_for_run = services.clone(); let executor = tokio::spawn(async move { - execute_claim( - &services_for_run, - LoopMode::Worker, - claim, - &shutdown, - Arc::new(WorkerHealth::default()), - ) - .await + execute_claim(&services_for_run, LoopMode::Drain, claim, &shutdown).await }); tokio::time::sleep(Duration::from_millis(10)).await; cancel.cancel(); @@ -1928,15 +1765,4 @@ mod tests { } } } - - #[tokio::test] - async fn shutdown_token_can_interrupt_idle_worker_sleep() { - let token = CancellationToken::new(); - token.cancel(); - let woke = tokio::select! { - _ = token.cancelled() => true, - _ = tokio::time::sleep(Duration::from_secs(30)) => false, - }; - assert!(woke); - } } diff --git a/crates/buzz-relay/src/api/mesh_demo.rs b/crates/buzz-relay/src/api/mesh_demo.rs index 902366f0d1..8649b97671 100644 --- a/crates/buzz-relay/src/api/mesh_demo.rs +++ b/crates/buzz-relay/src/api/mesh_demo.rs @@ -310,28 +310,13 @@ mod tests { owner_runtime, ); let from = hello.sender; - let mut inbound = router.accept_inbound(from, hello, stream).await.unwrap(); - let frame = inbound - .stream - .recv_validated(&echo_directory) - .await - .unwrap() - .unwrap(); - let ReliableFrame::Data(payload) = frame else { - panic!("expected data"); - }; - let community = inbound.stream.community_id().unwrap(); - inbound - .stream - .send_bytes(community, &payload) - .await - .unwrap(); - // Keep both endpoints and the owner stream alive until the caller has - // consumed the echo. Dropping the last endpoint clone closes the QUIC - // connection and used to make this production-path regression test - // deterministically time out before the return frame was read. - let _owner_endpoint_guard = accept_endpoint; - tokio::time::sleep(ECHO_TIMEOUT).await; + let inbound = router.accept_inbound(from, hello, stream).await.unwrap(); + crate::mesh_boot::run_demo_echo( + echo_directory, + inbound, + std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + ) + .await; }); // Forwarding side: join the same session through the demo core. @@ -356,7 +341,5 @@ mod tests { assert_eq!(body["outcome"], "forwarded"); assert_eq!(body["echoed_payload"], "mesh echo evidence"); owner_task.abort(); - drop(local_endpoint); - drop(owner_endpoint); } } diff --git a/crates/buzz-relay/src/handlers/side_effects.rs b/crates/buzz-relay/src/handlers/side_effects.rs index 68c73ab277..98f8a9aa84 100644 --- a/crates/buzz-relay/src/handlers/side_effects.rs +++ b/crates/buzz-relay/src/handlers/side_effects.rs @@ -904,27 +904,6 @@ pub async fn emit_membership_notification( target_pubkey: &[u8], actor_pubkey: &[u8], notification_kind: u32, -) -> anyhow::Result<()> { - emit_membership_notification_with_nonce( - tenant, - state, - channel_id, - target_pubkey, - actor_pubkey, - notification_kind, - None, - ) - .await -} - -async fn emit_membership_notification_with_nonce( - tenant: &TenantContext, - state: &Arc, - channel_id: Uuid, - target_pubkey: &[u8], - actor_pubkey: &[u8], - notification_kind: u32, - nonce: Option<&str>, ) -> anyhow::Result<()> { let target_hex = hex::encode(target_pubkey); let actor_hex = hex::encode(actor_pubkey); @@ -952,15 +931,8 @@ async fn emit_membership_notification_with_nonce( }) .to_string(); - let mut tags = vec![p_tag, h_tag]; - if let Some(nonce) = nonce { - tags.push( - Tag::parse(["nonce", nonce]) - .map_err(|e| anyhow::anyhow!("failed to build nonce tag: {e}"))?, - ); - } let event = EventBuilder::new(Kind::Custom(notification_kind as u16), content) - .tags(tags) + .tags([p_tag, h_tag]) .sign_with_keys(&state.relay_keypair) .map_err(|e| anyhow::anyhow!("failed to sign membership notification: {e}"))?; @@ -1633,21 +1605,28 @@ async fn handle_edit_metadata( // Resubscribe connected agents after restore: archiving evicts their // live subscriptions (CLOSED "channel access revoked") and unarchive // otherwise emits no signal that makes a connected agent resubscribe. - // We reuse member_added (44100) as the existing global resubscribe - // trigger. Each unarchive carries the accepted edit event id as a - // nonce so it cannot collide with the original member-added event (or - // another unarchive) within Nostr's one-second timestamp precision. + // We reuse the member_added notification (44100) purely as a resubscribe + // trigger — no membership actually changed here — because it flows on the + // agent's always-live global membership subscription, the same path + // remove/re-add uses to recover. Humans self-heal via the re-emitted + // kind:39000 discovery, so this is intentionally agent-scoped. + // + // Known limitation: emit_membership_notification builds a created_at=now + // event with no nonce, and insert_event skips fan-out on a duplicate id. + // Four sub-second toggles (archive->unarchive->archive->unarchive) on the + // same channel by the same actor could collide ids and skip a fan-out. + // Not reachable in practice — unarchive has a single human-driven caller; + // the reaper only auto-archives — so we don't engineer around it. for member in state.db.get_members(tenant.community(), channel_id).await? { - if let Err(e) = emit_membership_notification_with_nonce( + if let Err(e) = emit_membership_notification( tenant, state, channel_id, &member.pubkey, &actor_bytes, KIND_MEMBER_ADDED_NOTIFICATION, - Some(&event.id.to_hex()), ) .await { diff --git a/crates/buzz-test-client/tests/e2e_git.rs b/crates/buzz-test-client/tests/e2e_git.rs index 6b73fe2e77..3c82e31764 100644 --- a/crates/buzz-test-client/tests/e2e_git.rs +++ b/crates/buzz-test-client/tests/e2e_git.rs @@ -189,62 +189,16 @@ impl GitS3Probe { Self { bucket } } - fn configured_pointer_key(owner: &str, repo: &str) -> Option { + fn pointer_key(owner: &str, repo: &str) -> String { let repo = repo.strip_suffix(".git").unwrap_or(repo); - std::env::var("BUZZ_E2E_GIT_COMMUNITY_ID") - .ok() - .map(|community| format!("repos/{community}/{owner}/{repo}/pointer")) - } - - async fn find_pointer_key(&self, owner: &str, repo: &str) -> Option { - if let Some(key) = Self::configured_pointer_key(owner, repo) { - return Some(key); - } - - // Git pointers are community-scoped. Discover the exact pointer created - // for this random owner/repo rather than probing the obsolete unscoped - // `repos/{owner}/{repo}/pointer` layout. The optional env override above - // remains useful when a harness already knows the tenant UUID. - let repo = repo.strip_suffix(".git").unwrap_or(repo); - let suffix = format!("/{owner}/{repo}/pointer"); - let mut continuation = None; - let mut matches = Vec::new(); - loop { - let (page, _) = self - .bucket - .list_page( - "repos/".to_string(), - None, - continuation.take(), - None, - Some(1000), - ) - .await - .expect("list Git pointer namespace"); - matches.extend( - page.contents - .into_iter() - .map(|object| object.key) - .filter(|key| key.ends_with(&suffix)), - ); - if !page.is_truncated { - break; - } - continuation = page.next_continuation_token; - assert!( - continuation.is_some(), - "truncated Git pointer listing must include a continuation token" - ); + if let Ok(community) = std::env::var("BUZZ_E2E_GIT_COMMUNITY_ID") { + return format!("repos/{community}/{owner}/{repo}/pointer"); } - assert!( - matches.len() <= 1, - "random E2E owner/repo resolved multiple community pointers: {matches:?}" - ); - matches.pop() + format!("repos/{owner}/{repo}/pointer") } async fn pointer(&self, owner: &str, repo: &str) -> Option { - let key = self.find_pointer_key(owner, repo).await?; + let key = Self::pointer_key(owner, repo); match self.bucket.get_object(&key).await { Ok(resp) => { let etag = resp @@ -271,9 +225,10 @@ impl GitS3Probe { } tokio::time::sleep(Duration::from_millis(250)).await; } - let expected = Self::configured_pointer_key(owner, repo) - .unwrap_or_else(|| format!("repos//{owner}/{repo}/pointer")); - panic!("S3 manifest pointer {expected} never appeared") + panic!( + "S3 manifest pointer {} never appeared; git may have fallen back to disk", + Self::pointer_key(owner, repo) + ); } async fn assert_manifest_exists(&self, digest: &str) { diff --git a/crates/buzz-test-client/tests/e2e_media_extended.rs b/crates/buzz-test-client/tests/e2e_media_extended.rs index 156bfc7ffe..955bd9d6c4 100644 --- a/crates/buzz-test-client/tests/e2e_media_extended.rs +++ b/crates/buzz-test-client/tests/e2e_media_extended.rs @@ -19,11 +19,6 @@ fn relay_ws_url() -> String { .replace("https://", "wss://") } -fn relay_authority() -> String { - let url = url::Url::parse(&relay_http_url()).expect("relay HTTP URL"); - url[url::Position::BeforeHost..url::Position::AfterPort].to_string() -} - fn http_client() -> Client { Client::builder() .timeout(Duration::from_secs(15)) @@ -103,13 +98,15 @@ fn tiny_jpeg() -> Vec { } fn tiny_png() -> Vec { - // Valid, metadata-free 2x2 red PNG. + // Valid 2x2 red PNG generated by ffmpeg vec![ 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, 0x08, 0x02, 0x00, 0x00, 0x00, 0xfd, - 0xd4, 0x9a, 0x73, 0x00, 0x00, 0x00, 0x10, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0xf8, - 0xcf, 0xc0, 0x00, 0x44, 0x0c, 0x10, 0x0a, 0x00, 0x1f, 0xee, 0x03, 0xfd, 0x8b, 0x5f, 0x14, - 0xd4, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + 0xd4, 0x9a, 0x73, 0x00, 0x00, 0x00, 0x09, 0x70, 0x48, 0x59, 0x73, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x4f, 0x25, 0xc4, 0xd6, 0x00, 0x00, 0x00, 0x10, 0x49, 0x44, + 0x41, 0x54, 0x78, 0x9c, 0x63, 0xfc, 0xc3, 0x00, 0x02, 0x2c, 0x60, 0x92, 0x01, 0x00, 0x0d, + 0x04, 0x01, 0x02, 0xbf, 0x50, 0x15, 0xb3, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, + 0xae, 0x42, 0x60, 0x82, ] } @@ -374,7 +371,7 @@ async fn test_auth_server_tag_correct() { Tag::parse(["t", "upload"]).unwrap(), Tag::parse(["x", &sha256]).unwrap(), Tag::parse(["expiration", &(now + 300).to_string()]).unwrap(), - Tag::parse(["server", &relay_authority()]).unwrap(), + Tag::parse(["server", "localhost:3000"]).unwrap(), ], ); let resp = upload_with_auth(&client, &auth, &sha256, &jpeg).await; @@ -561,10 +558,6 @@ async fn test_ws_valid_imeta() { let resp = upload(&http, &keys, &jpeg).await; assert_eq!(resp.status(), 200); - let upload_desc: serde_json::Value = resp.json().await.expect("upload descriptor"); - let media_url = upload_desc["url"].as_str().expect("uploaded media URL"); - let media_size = upload_desc["size"].as_u64().expect("uploaded media size"); - // Connect via WebSocket let mut client = BuzzTestClient::connect(&relay_ws_url(), &keys) .await @@ -576,10 +569,10 @@ async fn test_ws_valid_imeta() { Tag::parse(["h", &channel_id]).unwrap(), Tag::parse([ "imeta", - &format!("url {media_url}"), + &format!("url http://localhost:3000/media/{sha256}.jpg"), "m image/jpeg", &format!("x {sha256}"), - &format!("size {media_size}"), + "size 347", ]) .unwrap(), ]) diff --git a/crates/buzz-test-client/tests/e2e_relay.rs b/crates/buzz-test-client/tests/e2e_relay.rs index a8adcc15ef..5b9a50b5b1 100644 --- a/crates/buzz-test-client/tests/e2e_relay.rs +++ b/crates/buzz-test-client/tests/e2e_relay.rs @@ -120,12 +120,7 @@ async fn seed_relay_member(host: &str, keys: &Keys, role: &str) { } async fn seed_relay_owner(keys: &Keys) { - seed_relay_member(&relay_authority(), keys, "owner").await; -} - -fn relay_authority() -> String { - let url = url::Url::parse(&relay_http_url()).expect("relay HTTP URL"); - url[url::Position::BeforeHost..url::Position::AfterPort].to_string() + seed_relay_member("localhost:3000", keys, "owner").await; } fn http_origin_for_host(host: &str) -> String { @@ -320,7 +315,7 @@ async fn test_invite_claim_rejects_invalid_code() { #[ignore] async fn test_invite_mint_requires_owner_or_admin() { let member = Keys::generate(); - seed_relay_member(&relay_authority(), &member, "member").await; + seed_relay_member("localhost:3000", &member, "member").await; let response = invite_post(&member, "/api/invites", "{}").await; assert_eq!(response.status(), reqwest::StatusCode::FORBIDDEN); @@ -796,10 +791,10 @@ async fn test_auth_event_kind_rejected() { /// NIP-11 max_subscriptions must be enforced; (limit+1)th REQ gets CLOSED. /// -/// This is a protocol-cap test, not an admission-throughput test. Open one REQ -/// at a time and wait out any shared fixed-window quota before retrying a REQ -/// rejected specifically as `rate-limited`, so production admission remains -/// enabled while the test deterministically reaches the independent 1024 cap. +/// The relay's MAX_SUBSCRIPTIONS is 1024. Opening 1024 subs in a test is slow, +/// so we open a smaller batch and verify the NIP-11 advertised limit matches +/// the actual enforcement constant. The full-limit test is covered by the +/// NIP-11 assertion below (which verifies the advertised value is 1024). #[tokio::test] #[ignore] async fn test_subscription_limit_enforced() { @@ -807,73 +802,58 @@ async fn test_subscription_limit_enforced() { let keys = Keys::generate(); let mut client = BuzzTestClient::connect(&url, &keys).await.expect("connect"); + // Open 1024 subscriptions (the relay's MAX_SUBSCRIPTIONS). for i in 0..1024 { let sid = format!("limit-sub-{i}"); - let filter = Filter::new().kind(Kind::Custom(49_999)); - subscribe_until_eose(&mut client, &sid, filter).await; - } - - let overflow_sid = sub_id("overflow"); - let filter = Filter::new().kind(Kind::Custom(49_999)); - loop { + let filter = Filter::new().kind(Kind::Custom(9)); client - .subscribe(&overflow_sid, vec![filter.clone()]) + .subscribe(&sid, vec![filter]) .await - .expect("send overflow REQ"); - - match client - .recv_event(Duration::from_secs(6)) + .expect("subscribe"); + // Drain EOSE to avoid buffer buildup. + client + .collect_until_eose(&sid, Duration::from_secs(5)) .await - .expect("recv overflow CLOSED") - { - RelayMessage::Closed { - subscription_id, - message, - } if subscription_id == overflow_sid && message.starts_with("rate-limited:") => { - tokio::time::sleep(Duration::from_secs(5)).await; - } - RelayMessage::Closed { - subscription_id, - message, - } => { - assert_eq!(subscription_id, overflow_sid); - assert!( - message.to_lowercase().contains("too many"), - "Expected 'too many' in CLOSED message, got: {message}" - ); - break; - } - other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), - } + .expect("EOSE"); } - client.disconnect().await.expect("disconnect"); -} + let overflow_sid = sub_id("overflow"); + // Use a kind that no other test writes, so we don't receive stale events. + let filter = Filter::new().kind(Kind::Custom(49999)); + client + .subscribe(&overflow_sid, vec![filter]) + .await + .expect("send REQ"); -async fn subscribe_until_eose(client: &mut BuzzTestClient, sid: &str, filter: Filter) { - loop { - client - .subscribe(sid, vec![filter.clone()]) - .await - .expect("subscribe"); - match client - .recv_event(Duration::from_secs(6)) + // Drain EOSE and stale events from the 100 earlier subscriptions + // until we receive the CLOSED for the overflow subscription. + let msg = loop { + let m = client + .recv_event(Duration::from_secs(5)) .await - .expect("EOSE or rate-limit CLOSED") - { - RelayMessage::Eose { subscription_id } => { - assert_eq!(subscription_id, sid); - return; - } - RelayMessage::Closed { - subscription_id, - message, - } if subscription_id == sid && message.starts_with("rate-limited:") => { - tokio::time::sleep(Duration::from_secs(5)).await; - } - other => panic!("unexpected response while opening {sid}: {other:?}"), + .expect("recv CLOSED (or timeout)"); + match &m { + RelayMessage::Eose { .. } => continue, + RelayMessage::Event { .. } => continue, // stale event from earlier subs + _ => break m, } + }; + + match msg { + RelayMessage::Closed { + subscription_id, + message, + } => { + assert_eq!(subscription_id, overflow_sid); + assert!( + message.to_lowercase().contains("too many"), + "Expected 'too many' in CLOSED message, got: {message}" + ); + } + other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } + + client.disconnect().await.expect("disconnect"); } #[tokio::test] From 76224c54316474e97bfc02de34c03be29dda45bf Mon Sep 17 00:00:00 2001 From: Kalvin Chau Date: Mon, 3 Aug 2026 11:09:58 -0700 Subject: [PATCH 22/51] feat: scale community deletion to large tenant buckets replace the full-bucket inline inventory with prefix-scoped enumeration and a chunked side-table manifest: - inventory lists only the three tenant prefixes (_meta/, _uploads/, repos/); unknown-key detection moves to a decoupled fleet-wide taxonomy sweep (new deletions sweep command) gated on freshness at submit/approve - destructive freeze stores per-prefix count/bytes/sha256-digest summaries on the request row and streams full key lists into community_deletion_manifest_keys (~10k keys/row); freeze validates chunks against the frozen digests in the same tx - drained stage deletes via batched s3 DeleteObjects with per-chunk checkpoints; NoSuchKey/NoSuchVersion map to already-missing for idempotent resume; versioning artifacts fail permanent - per-key HEAD verification replaced by a bucket-level versioning probe and a final empty-prefix listing check - fenced stage reordered: serving-write drain is verified before the destructive freeze, closing the late-write window - chunk rows are trigger-guarded (immutable except the deleted_at stamp; delete only pre-freeze or terminal) and purged at logically_verified; schema folded into unmerged 0027 - record_taxonomy_sweep clamps completed_at server-side so sweeper clock skew cannot violate the completed_at >= started_at check env knobs: BUZZ_DELETION_STORAGE_MAX_OBJECTS (per community), BUZZ_DELETION_SWEEP_MAX_OBJECTS, BUZZ_DELETION_SWEEP_MAX_AGE_SECS, BUZZ_DELETION_MANIFEST_CHUNK_KEYS Signed-off-by: Kalvin Chau --- Cargo.lock | 1 + crates/buzz-db/src/deletion.rs | 845 +++++++++++++++++---- crates/buzz-db/src/migration.rs | 2 + crates/buzz-deletion/Cargo.toml | 1 + crates/buzz-deletion/src/lib.rs | 765 +++++++++++-------- crates/buzz-media/src/bucket_index.rs | 284 ++++--- crates/buzz-media/src/lib.rs | 9 +- crates/buzz-media/src/storage.rs | 228 +++--- crates/buzz-relay/src/api/git/transport.rs | 27 +- crates/buzz-relay/src/handlers/ingest.rs | 27 +- migrations/0028_community_deletion.sql | 78 ++ 11 files changed, 1576 insertions(+), 691 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4e05812cc5..6e064b2e5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1028,6 +1028,7 @@ dependencies = [ "buzz-core", "buzz-db", "buzz-media", + "chrono", "clap", "deadpool-redis", "hex", diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index caefd86678..1ea3eac27b 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -275,37 +275,124 @@ pub struct SchemaManifest { pub fenced_tables: Vec, } -/// Frozen storage inventory supplied by the object-store adapter. +/// Frozen storage inventory supplied by the object-store adapter: slim +/// per-prefix summaries bound to a clean fleet taxonomy sweep. The concrete +/// key list never lives on the request row — the destructive freeze persists +/// it as chunked `community_deletion_manifest_keys` rows that must hash to +/// these digests. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct StorageManifest { /// Adapter schema version. pub version: i32, - /// Every tenant-owned key that will be removed, sorted. - pub tenant_keys: Vec, - /// Immutable observations used to detect replacement under an approved key. - #[serde(default)] - pub tenant_objects: Vec, - /// Git pointer keys among tenant keys. - pub git_pointer_keys: Vec, - /// Media sidecar keys among tenant keys. - pub media_sidecar_keys: Vec, - /// Media upload-record keys among tenant keys. - pub media_upload_keys: Vec, - /// Fleet keys unknown to the current taxonomy. Non-empty fails inventory. + /// Per-prefix frozen summaries, strictly sorted by prefix. + pub prefixes: Vec, + /// Clean fleet taxonomy sweep this inventory relied on. + pub taxonomy_sweep_id: Uuid, + /// Tenant-prefix keys outside the exact writer taxonomy. Non-empty fails. pub unknown_keys: Vec, - /// Keys whose current object version cannot be safely removed. Non-empty fails. - pub unsupported_version_keys: Vec, } -/// Frozen observation of one tenant-owned object binding. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] -pub struct StorageObject { - /// Exact bucket key. - pub key: String, - /// Current object size. - pub size: u64, - /// Opaque entity tag used to detect object replacement. - pub e_tag: Option, +/// Frozen summary of one community-scoped key prefix. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PrefixManifest { + /// Exact community-scoped listing prefix. + pub prefix: String, + /// Objects under the prefix at enumeration time. + pub object_count: u64, + /// Total object bytes under the prefix at enumeration time. + pub total_bytes: u64, + /// Hex SHA-256 of the newline-terminated ascending key stream. + pub keys_digest: String, +} + +/// One frozen chunk of the destructive key list. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ManifestKeyChunk { + /// Position in the frozen chunk sequence. + pub chunk_no: i64, + /// The tenant prefix every key in this chunk lives under. + pub prefix: String, + /// Strictly ascending keys. + pub keys: Vec, +} + +/// One durable fleet-wide object-store taxonomy sweep record. +#[derive(Debug, Clone, Serialize)] +pub struct TaxonomySweep { + /// Sweep identity bound into storage manifests. + pub id: Uuid, + /// Listing start time. + pub started_at: DateTime, + /// Record time. + pub completed_at: DateTime, + /// Total objects listed. + pub listed_objects: i64, + /// Exact count of keys outside the known writer taxonomy. + pub unknown_object_count: i64, + /// Bounded sample of unknown keys. + pub unknown_key_sample: Vec, + /// Fleet object cap the sweep ran under. + pub object_cap: i64, +} + +type TaxonomySweepRow = ( + Uuid, + DateTime, + DateTime, + i64, + i64, + sqlx::types::Json>, + i64, +); + +/// Streaming SHA-256 over a strictly ascending key stream. +/// +/// The executor's prefix enumeration and the destructive freeze's chunk +/// validation both fold keys through this, so "the chunk rows are exactly +/// the frozen enumeration" reduces to digest equality. Each key is hashed +/// with a trailing newline so concatenation cannot alias two streams. +pub struct KeyStreamDigest { + hasher: Sha256, + last: Option, + count: u64, +} + +impl Default for KeyStreamDigest { + fn default() -> Self { + Self::new() + } +} + +impl KeyStreamDigest { + /// Start an empty stream. + pub fn new() -> Self { + Self { + hasher: Sha256::new(), + last: None, + count: 0, + } + } + + /// Fold the next key. Keys must arrive strictly ascending — S3 + /// `ListObjectsV2` order — so one out-of-order or duplicate key fails + /// closed instead of silently producing a different digest. + pub fn fold(&mut self, key: &str) -> Result<()> { + if self.last.as_deref().is_some_and(|last| last >= key) { + return Err(DbError::DeletionSafety(format!( + "storage key stream is not strictly ascending at {key}" + ))); + } + self.hasher.update(key.as_bytes()); + self.hasher.update(b"\n"); + self.last = Some(key.to_owned()); + self.count += 1; + Ok(()) + } + + /// Hex digest and key count of everything folded. + pub fn finish(self) -> (String, u64) { + (hex::encode(self.hasher.finalize()), self.count) + } } /// Full frozen inventory approved at the destructive boundary. @@ -1149,6 +1236,18 @@ impl DeletionStore { let mut tx = self.pool.begin().await?; verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; validate_storage_manifest(manifest)?; + // The chunk rows are the concrete delete list; the freeze commits only + // if they hash to the manifest's frozen per-prefix digests. Loading the + // full chunk stream is a one-time freeze-boundary cost bounded by the + // per-community object cap, not the fleet. + let chunks: Vec<(i64, String, sqlx::types::Json>)> = sqlx::query_as( + "SELECT chunk_no, prefix, keys FROM community_deletion_manifest_keys \ + WHERE request_id = $1 ORDER BY chunk_no", + ) + .bind(token.request_id) + .fetch_all(&mut *tx) + .await?; + validate_manifest_key_chunks(manifest, &chunks)?; let affected = sqlx::query( "UPDATE community_deletion_requests \ SET destructive_storage_manifest = COALESCE(destructive_storage_manifest, $4), \ @@ -1176,6 +1275,185 @@ impl DeletionStore { Ok(()) } + /// Remove key chunks left by an interrupted destructive freeze. + /// + /// The chunk-table guard rejects this once the destructive manifest has + /// frozen, so a retried freeze can only rewrite chunks that were never + /// bound to a committed manifest. + pub async fn clear_manifest_key_chunks(&self, token: &LeaseToken) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(token.request_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Append one immutable chunk of the destructive key list. + pub async fn append_manifest_key_chunk( + &self, + token: &LeaseToken, + chunk_no: i64, + prefix: &str, + keys: &[String], + ) -> Result<()> { + if keys.is_empty() { + return Err(DbError::DeletionSafety( + "refusing to persist an empty manifest key chunk".to_string(), + )); + } + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; + sqlx::query( + "INSERT INTO community_deletion_manifest_keys \ + (request_id, chunk_no, prefix, keys) VALUES ($1, $2, $3, $4)", + ) + .bind(token.request_id) + .bind(chunk_no) + .bind(prefix) + .bind(sqlx::types::Json(keys)) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Return the next frozen chunk not yet confirmed deleted, in chunk order. + pub async fn next_pending_manifest_chunk( + &self, + token: &LeaseToken, + ) -> Result> { + let row: Option<(i64, String, sqlx::types::Json>)> = sqlx::query_as( + "SELECT chunk_no, prefix, keys FROM community_deletion_manifest_keys \ + WHERE request_id = $1 AND deleted_at IS NULL ORDER BY chunk_no LIMIT 1", + ) + .bind(token.request_id) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|(chunk_no, prefix, keys)| ManifestKeyChunk { + chunk_no, + prefix, + keys: keys.0, + })) + } + + /// Return `(total, deleted)` chunk counts for one request. + pub async fn manifest_chunk_progress(&self, request_id: Uuid) -> Result<(i64, i64)> { + sqlx::query_as( + "SELECT count(*), count(deleted_at) FROM community_deletion_manifest_keys \ + WHERE request_id = $1", + ) + .bind(request_id) + .fetch_one(&self.pool) + .await + .map_err(Into::into) + } + + /// Stamp one chunk's keys durably removed and checkpoint it atomically. + pub async fn mark_manifest_chunk_deleted( + &self, + token: &LeaseToken, + chunk_no: i64, + detail: serde_json::Value, + ) -> Result<()> { + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::Drained, generation).await?; + let affected = sqlx::query( + "UPDATE community_deletion_manifest_keys SET deleted_at = now() \ + WHERE request_id = $1 AND chunk_no = $2 AND deleted_at IS NULL", + ) + .bind(token.request_id) + .bind(chunk_no) + .execute(&mut *tx) + .await? + .rows_affected(); + if affected != 1 { + return Err(DbError::DeletionSafety(format!( + "manifest key chunk {chunk_no} is missing or already stamped for request {}", + token.request_id + ))); + } + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::Drained, + &format!("chunk:{chunk_no}"), + detail, + ) + .await?; + tx.commit().await?; + Ok(()) + } + + /// Record one completed fleet-wide taxonomy sweep. + pub async fn record_taxonomy_sweep( + &self, + started_at: DateTime, + listed_objects: u64, + unknown_object_count: u64, + unknown_key_sample: &[String], + object_cap: u64, + ) -> Result { + let listed = i64::try_from(listed_objects) + .map_err(|_| DbError::DeletionSafety("sweep object count overflow".to_string()))?; + let unknown = i64::try_from(unknown_object_count) + .map_err(|_| DbError::DeletionSafety("sweep unknown count overflow".to_string()))?; + let cap = i64::try_from(object_cap) + .map_err(|_| DbError::DeletionSafety("sweep object cap overflow".to_string()))?; + // completed_at is clamped server-side: the sweeper's clock supplies + // started_at, and skew ahead of Postgres would otherwise violate the + // completed_at >= started_at check. + let (id, completed_at): (Uuid, DateTime) = sqlx::query_as( + "INSERT INTO storage_taxonomy_sweeps \ + (started_at, completed_at, listed_objects, unknown_object_count, \ + unknown_key_sample, object_cap) \ + VALUES ($1, GREATEST(now(), $1), $2, $3, $4, $5) RETURNING id, completed_at", + ) + .bind(started_at) + .bind(listed) + .bind(unknown) + .bind(sqlx::types::Json(unknown_key_sample)) + .bind(cap) + .fetch_one(&self.pool) + .await?; + Ok(TaxonomySweep { + id, + started_at, + completed_at, + listed_objects: listed, + unknown_object_count: unknown, + unknown_key_sample: unknown_key_sample.to_vec(), + object_cap: cap, + }) + } + + /// Return the most recently completed taxonomy sweep, if any. + pub async fn latest_taxonomy_sweep(&self) -> Result> { + let row: Option = sqlx::query_as( + "SELECT id, started_at, completed_at, listed_objects, unknown_object_count, \ + unknown_key_sample, object_cap \ + FROM storage_taxonomy_sweeps ORDER BY completed_at DESC LIMIT 1", + ) + .fetch_optional(&self.pool) + .await?; + Ok(row.map( + |(id, started_at, completed_at, listed, unknown, sample, cap)| TaxonomySweep { + id, + started_at, + completed_at, + listed_objects: listed, + unknown_object_count: unknown, + unknown_key_sample: sample.0, + object_cap: cap, + }, + )) + } + /// Return whether all pre-fence external side-effect leases have expired or released. pub async fn serving_writes_drained(&self, community: CommunityId) -> Result { sqlx::query_scalar( @@ -1225,57 +1503,6 @@ impl DeletionStore { Ok(()) } - /// Record one completed object-binding deletion checkpoint. - pub async fn checkpoint_storage_object_removed( - &self, - token: &LeaseToken, - key: &str, - already_missing: bool, - ) -> Result<()> { - let generation = require_fence_generation(token)?; - let mut tx = self.pool.begin().await?; - verify_lease_and_fence(&mut tx, token, DeletionStage::Drained, generation).await?; - checkpoint_completed_tx( - &mut tx, - token, - DeletionStage::Drained, - &storage_checkpoint_key(key), - serde_json::json!({"key": key, "already_missing": already_missing}), - ) - .await?; - tx.commit().await?; - Ok(()) - } - - /// Return object keys already durably removed in the Drained stage. - pub async fn completed_storage_object_keys( - &self, - token: &LeaseToken, - ) -> Result> { - let rows: Vec = sqlx::query_scalar( - "SELECT detail FROM community_deletion_checkpoints \ - WHERE request_id = $1 AND stage = 'drained' AND status = 'completed' \ - AND unit_key LIKE 'object:%' ORDER BY sequence", - ) - .bind(token.request_id) - .fetch_all(&self.pool) - .await?; - rows.into_iter() - .map(|detail| { - detail - .get("key") - .and_then(serde_json::Value::as_str) - .map(str::to_owned) - .ok_or_else(|| { - DbError::DeletionSafety(format!( - "malformed storage checkpoint for deletion {}", - token.request_id - )) - }) - }) - .collect() - } - /// Mark storage binding removal after adapter verification. pub async fn mark_bindings_removed( &self, @@ -1428,20 +1655,44 @@ impl DeletionStore { Ok(()) } - /// Commit the cross-store logical verification checkpoint. + /// Commit the cross-store logical verification checkpoint and drop the + /// frozen key chunks in the same transaction. + /// + /// The chunk rows are working data, not audit evidence — per-prefix + /// counts, digests, and checkpoint history stay on the request row, and + /// the raw key list of a deleted community should not be retained. + /// Blocked requests never reach this transition, so their chunks survive + /// for resumption or operator inspection. pub async fn mark_logically_verified( &self, token: &LeaseToken, detail: serde_json::Value, ) -> Result<()> { - self.advance_with_checkpoint( + let generation = require_fence_generation(token)?; + let mut tx = self.pool.begin().await?; + verify_lease_and_fence(&mut tx, token, DeletionStage::CachePurged, generation).await?; + advance_request_tx( + &mut tx, token, DeletionStage::CachePurged, DeletionStage::LogicallyVerified, + Some(generation), + ) + .await?; + checkpoint_completed_tx( + &mut tx, + token, + DeletionStage::CachePurged, "verify_cross_store_absence", detail, ) - .await + .await?; + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(token.request_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) } /// Finish logical deletion and enter the physical-expiry pending state. @@ -1925,45 +2176,50 @@ fn validate_serving_migration_version(migration_version: Option) -> Result< } } -/// Fail closed when storage inventory reports unknown or unsupported data. +/// Fail closed when storage inventory reports unknown or malformed data. pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { - if manifest.version != 2 { + if manifest.version != 3 { return Err(DbError::DeletionSafety(format!( "unsupported storage manifest version {}", manifest.version ))); } - let object_keys = manifest - .tenant_objects - .iter() - .map(|object| object.key.as_str()) - .collect::>(); - let tenant_keys = manifest - .tenant_keys - .iter() - .map(String::as_str) - .collect::>(); - if object_keys != tenant_keys { + if manifest.prefixes.is_empty() { return Err(DbError::DeletionSafety( - "storage manifest object observations do not exactly match tenant keys".to_string(), + "storage manifest has no tenant prefixes".to_string(), )); } if manifest - .tenant_objects - .iter() - .any(|object| object.e_tag.is_none()) + .prefixes + .windows(2) + .any(|pair| pair[0].prefix >= pair[1].prefix) { return Err(DbError::DeletionSafety( - "storage manifest contains a tenant object without ETag".to_string(), + "storage manifest prefixes are not strictly sorted".to_string(), )); } - if manifest - .tenant_objects - .windows(2) - .any(|pair| pair[0].key >= pair[1].key) - { + for prefix in &manifest.prefixes { + // An empty prefix would enumerate — and delete — the whole bucket. + if prefix.prefix.is_empty() { + return Err(DbError::DeletionSafety( + "storage manifest contains an empty prefix".to_string(), + )); + } + if prefix.keys_digest.len() != 64 + || !prefix + .keys_digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(DbError::DeletionSafety(format!( + "storage manifest digest for {} is not lowercase hex sha-256", + prefix.prefix + ))); + } + } + if manifest.taxonomy_sweep_id.is_nil() { return Err(DbError::DeletionSafety( - "storage manifest tenant objects are not strictly sorted".to_string(), + "storage manifest is not bound to a taxonomy sweep".to_string(), )); } if !manifest.unknown_keys.is_empty() { @@ -1972,11 +2228,68 @@ pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { manifest.unknown_keys.join(",") ))); } - if !manifest.unsupported_version_keys.is_empty() { - return Err(DbError::DeletionSafety(format!( - "unsupported object versions block deletion: {}", - manifest.unsupported_version_keys.join(",") - ))); + Ok(()) +} + +/// Verify the persisted chunk stream is exactly the frozen enumeration: +/// contiguous chunk numbers, chunks grouped by manifest prefix order, every +/// key under its chunk's prefix, and per-prefix digest/count equality. +fn validate_manifest_key_chunks( + manifest: &StorageManifest, + chunks: &[(i64, String, sqlx::types::Json>)], +) -> Result<()> { + let close = |summary: &PrefixManifest, digest: KeyStreamDigest| -> Result<()> { + let (hex_digest, count) = digest.finish(); + if hex_digest != summary.keys_digest || count != summary.object_count { + return Err(DbError::DeletionSafety(format!( + "frozen key chunks do not match the destructive manifest for prefix {}", + summary.prefix + ))); + } + Ok(()) + }; + let mut remaining = manifest.prefixes.iter(); + let mut current = remaining.next(); + let mut digest = KeyStreamDigest::new(); + for (index, (chunk_no, chunk_prefix, keys)) in chunks.iter().enumerate() { + if *chunk_no != i64::try_from(index).unwrap_or(i64::MAX) { + return Err(DbError::DeletionSafety( + "frozen key chunk sequence has gaps".to_string(), + )); + } + loop { + match current { + Some(summary) if summary.prefix == *chunk_prefix => break, + Some(summary) => { + close(summary, std::mem::take(&mut digest))?; + current = remaining.next(); + } + None => { + return Err(DbError::DeletionSafety(format!( + "frozen key chunk prefix {chunk_prefix} is not in the destructive manifest" + ))); + } + } + } + if keys.0.is_empty() { + return Err(DbError::DeletionSafety( + "frozen key chunk is empty".to_string(), + )); + } + for key in &keys.0 { + if !key.starts_with(chunk_prefix.as_str()) { + return Err(DbError::DeletionSafety(format!( + "frozen key {key} is outside its chunk prefix {chunk_prefix}" + ))); + } + digest.fold(key)?; + } + } + if let Some(summary) = current { + close(summary, digest)?; + } + for summary in remaining { + close(summary, KeyStreamDigest::new())?; } Ok(()) } @@ -2153,10 +2466,6 @@ async fn checkpoint_failed_tx( Ok(()) } -fn storage_checkpoint_key(key: &str) -> String { - format!("object:{}", hex::encode(Sha256::digest(key.as_bytes()))) -} - fn serialize_community_id( community: &CommunityId, serializer: S, @@ -2232,16 +2541,25 @@ fn bound_text(input: &str, max: usize) -> String { mod tests { use super::*; + fn empty_prefix(prefix: &str) -> PrefixManifest { + PrefixManifest { + prefix: prefix.to_string(), + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + } + } + fn storage_manifest() -> StorageManifest { StorageManifest { - version: 2, - tenant_keys: Vec::new(), - tenant_objects: Vec::new(), - git_pointer_keys: Vec::new(), - media_sidecar_keys: Vec::new(), - media_upload_keys: Vec::new(), + version: 3, + prefixes: vec![ + empty_prefix("_meta/c/"), + empty_prefix("_uploads/c/"), + empty_prefix("repos/c/"), + ], + taxonomy_sweep_id: Uuid::from_u128(1), unknown_keys: Vec::new(), - unsupported_version_keys: Vec::new(), } } @@ -2303,29 +2621,24 @@ mod tests { } #[test] - fn storage_manifest_rejects_missing_identity_observation() { - let mut manifest = storage_manifest(); - manifest.tenant_keys.push("tenant/key".to_string()); - assert!(validate_storage_manifest(&manifest).is_err()); - } + fn storage_manifest_shape_invariants_fail_closed() { + assert!(validate_storage_manifest(&storage_manifest()).is_ok()); - #[test] - fn storage_manifest_rejects_missing_object_etag() { - let mut manifest = storage_manifest(); - manifest.tenant_keys.push("tenant/key".to_string()); - manifest.tenant_objects.push(StorageObject { - key: "tenant/key".to_string(), - size: 1, - e_tag: None, - }); - assert!(validate_storage_manifest(&manifest).is_err()); - } + let mut unsorted = storage_manifest(); + unsorted.prefixes.swap(0, 1); + assert!(validate_storage_manifest(&unsorted).is_err()); - #[test] - fn object_checkpoint_keys_are_bounded_and_stable() { - let key = "x".repeat(4096); - assert_eq!(storage_checkpoint_key(&key), storage_checkpoint_key(&key)); - assert_eq!(storage_checkpoint_key(&key).len(), "object:".len() + 64); + let mut whole_bucket = storage_manifest(); + whole_bucket.prefixes[0].prefix = String::new(); + assert!(validate_storage_manifest(&whole_bucket).is_err()); + + let mut malformed_digest = storage_manifest(); + malformed_digest.prefixes[0].keys_digest = "not-hex".to_string(); + assert!(validate_storage_manifest(&malformed_digest).is_err()); + + let mut unswept = storage_manifest(); + unswept.taxonomy_sweep_id = Uuid::nil(); + assert!(validate_storage_manifest(&unswept).is_err()); } #[test] @@ -2337,13 +2650,64 @@ mod tests { } #[test] - fn unsupported_object_versions_fail_closed() { + fn key_stream_digest_requires_strict_order_and_is_chunking_invariant() { + let keys = ["a/1", "a/2", "a/3"]; + let mut whole = KeyStreamDigest::new(); + for key in keys { + whole.fold(key).expect("ascending fold"); + } + // The digest must not depend on where chunk boundaries fall. + let mut split = KeyStreamDigest::new(); + split.fold(keys[0]).expect("chunk one"); + split.fold(keys[1]).expect("chunk one"); + split.fold(keys[2]).expect("chunk two"); + assert_eq!(whole.finish(), split.finish()); + + let mut out_of_order = KeyStreamDigest::new(); + out_of_order.fold("b").expect("first key"); + assert!(out_of_order.fold("a").is_err()); + let mut duplicate = KeyStreamDigest::new(); + duplicate.fold("a").expect("first key"); + assert!(duplicate.fold("a").is_err()); + } + + #[test] + fn manifest_key_chunks_must_hash_to_the_frozen_summaries() { + let keys = vec!["_meta/c/1".to_string(), "_meta/c/2".to_string()]; + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold"); + } + let (hex_digest, count) = digest.finish(); let mut manifest = storage_manifest(); - manifest - .unsupported_version_keys - .push("_meta/community/blob.json".to_string()); - let error = validate_storage_manifest(&manifest).expect_err("object version must block"); - assert!(error.to_string().contains("unsupported object versions")); + manifest.prefixes[0].object_count = count; + manifest.prefixes[0].keys_digest = hex_digest; + + let chunk = |chunk_no: i64, keys: &[String]| { + ( + chunk_no, + "_meta/c/".to_string(), + sqlx::types::Json(keys.to_vec()), + ) + }; + assert!(validate_manifest_key_chunks( + &manifest, + &[chunk(0, &keys[..1]), chunk(1, &keys[1..])] + ) + .is_ok()); + // Missing, reordered, or extra keys change the digest. + assert!(validate_manifest_key_chunks(&manifest, &[chunk(0, &keys[..1])]).is_err()); + // A gap in the chunk sequence is an interrupted write, not a manifest. + assert!(validate_manifest_key_chunks(&manifest, &[chunk(1, &keys)]).is_err()); + // A key outside its chunk's prefix must never freeze. + let foreign = vec!["_uploads/other/1".to_string()]; + assert!( + validate_manifest_key_chunks(&manifest, &[chunk(0, &keys), chunk(1, &foreign)]) + .is_err() + ); + // No chunks at all only matches an all-empty manifest. + assert!(validate_manifest_key_chunks(&manifest, &[]).is_err()); + assert!(validate_manifest_key_chunks(&storage_manifest(), &[]).is_ok()); } #[test] @@ -2393,6 +2757,28 @@ mod postgres_tests { (db, store) } + fn empty_prefix_manifest(prefix: String) -> PrefixManifest { + PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + } + } + + fn empty_storage_manifest(community: CommunityId, sweep_id: Uuid) -> StorageManifest { + StorageManifest { + version: 3, + prefixes: vec![ + empty_prefix_manifest(format!("_meta/{community}/")), + empty_prefix_manifest(format!("_uploads/{community}/")), + empty_prefix_manifest(format!("repos/{community}/")), + ], + taxonomy_sweep_id: sweep_id, + unknown_keys: Vec::new(), + } + } + async fn inventoried_request( db: &Db, store: &DeletionStore, @@ -2407,21 +2793,16 @@ mod postgres_tests { .await .expect("submit"); assert_eq!(submitted.community_id, community.id); + let sweep = store + .record_taxonomy_sweep(Utc::now(), 0, 0, &[], 1_000_000) + .await + .expect("record clean taxonomy sweep"); let inventory = FrozenInventory { schema: store .inventory_schema(community.id) .await .expect("schema inventory"), - storage: StorageManifest { - version: 2, - tenant_keys: Vec::new(), - tenant_objects: Vec::new(), - git_pointer_keys: Vec::new(), - media_sidecar_keys: Vec::new(), - media_upload_keys: Vec::new(), - unknown_keys: Vec::new(), - unsupported_version_keys: Vec::new(), - }, + storage: empty_storage_manifest(community.id, sweep.id), }; let request = store .freeze_inventory(submitted.id, &inventory) @@ -2877,9 +3258,13 @@ mod postgres_tests { .await .expect("identical destructive manifest retry"); let mut drifted_storage = inventory.storage.clone(); - drifted_storage - .media_upload_keys - .push("media/drifted-after-fence".to_string()); + let mut drifted_digest = KeyStreamDigest::new(); + drifted_digest + .fold("media/drifted-after-fence") + .expect("fold drifted key"); + let (drifted_hex, drifted_count) = drifted_digest.finish(); + drifted_storage.prefixes[0].object_count = drifted_count; + drifted_storage.prefixes[0].keys_digest = drifted_hex; assert!(matches!( store .freeze_destructive_storage_manifest(&token, &drifted_storage) @@ -2965,4 +3350,148 @@ mod postgres_tests { .to_string() .contains("tombstones are permanent")); } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn manifest_key_chunks_bind_freeze_execution_and_cleanup() { + let (db, store) = store().await; + let (request, inventory) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("won claim"); + store.begin_quiescing(&claim.lease).await.expect("quiesce"); + let generation = store.fence(&claim.lease).await.expect("fence"); + let token = LeaseToken { + fence_generation: Some(generation), + ..claim.lease + }; + + let meta_prefix = format!("_meta/{}/", request.community_id); + let keys = vec![ + format!("{meta_prefix}{}.json", "a".repeat(64)), + format!("{meta_prefix}{}.json", "b".repeat(64)), + ]; + store + .append_manifest_key_chunk(&token, 0, &meta_prefix, &keys[..1]) + .await + .expect("append chunk 0"); + store + .append_manifest_key_chunk(&token, 1, &meta_prefix, &keys[1..]) + .await + .expect("append chunk 1"); + + // A manifest whose digests do not cover the chunk stream must not freeze. + assert!(matches!( + store + .freeze_destructive_storage_manifest(&token, &inventory.storage) + .await, + Err(DbError::DeletionSafety(_)) + )); + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold key"); + } + let (hex_digest, count) = digest.finish(); + let mut storage = inventory.storage.clone(); + storage.prefixes[0].object_count = count; + storage.prefixes[0].total_bytes = 2; + storage.prefixes[0].keys_digest = hex_digest; + store + .freeze_destructive_storage_manifest(&token, &storage) + .await + .expect("freeze manifest matching chunks"); + + // Frozen chunks are immutable working data until terminal cleanup. + assert!(sqlx::query( + "UPDATE community_deletion_manifest_keys SET keys = '[]'::jsonb \ + WHERE request_id = $1 AND chunk_no = 0", + ) + .bind(request.id) + .execute(&db.pool) + .await + .is_err()); + assert!( + sqlx::query("DELETE FROM community_deletion_manifest_keys WHERE request_id = $1") + .bind(request.id) + .execute(&db.pool) + .await + .is_err() + ); + assert!(store.clear_manifest_key_chunks(&token).await.is_err()); + + store.mark_drained(&token).await.expect("drained"); + let first = store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .expect("chunk 0 pending"); + assert_eq!(first.chunk_no, 0); + assert_eq!(first.keys, keys[..1]); + store + .mark_manifest_chunk_deleted(&token, 0, serde_json::json!({"deleted": 1})) + .await + .expect("stamp chunk 0"); + assert!( + matches!( + store + .mark_manifest_chunk_deleted(&token, 0, serde_json::json!({})) + .await, + Err(DbError::DeletionSafety(_)) + ), + "a chunk stamp is one-way" + ); + let second = store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .expect("chunk 1 pending after resume"); + assert_eq!(second.chunk_no, 1); + store + .mark_manifest_chunk_deleted(&token, 1, serde_json::json!({"deleted": 1})) + .await + .expect("stamp chunk 1"); + assert!(store + .next_pending_manifest_chunk(&token) + .await + .expect("pending chunk") + .is_none()); + assert_eq!( + store + .manifest_chunk_progress(request.id) + .await + .expect("progress"), + (2, 2) + ); + + store + .mark_bindings_removed(&token, serde_json::json!({"deleted_keys": 2})) + .await + .expect("bindings removed"); + store.purge_postgres(&token).await.expect("purge postgres"); + store + .mark_cache_purged(&token, serde_json::json!({"keys": 0})) + .await + .expect("cache purged"); + store + .verify_postgres_logically_deleted(&token) + .await + .expect("verify postgres"); + store + .mark_logically_verified(&token, serde_json::json!({"all": true})) + .await + .expect("logically verified"); + assert_eq!( + store + .manifest_chunk_progress(request.id) + .await + .expect("progress after terminal cleanup"), + (0, 0) + ); + } } diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 1fb4a02fac..ac3b564db9 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -351,6 +351,8 @@ mod tests { "community_deletion_requests", "community_deletion_approvals", "community_deletion_checkpoints", + "community_deletion_manifest_keys", + "storage_taxonomy_sweeps", "community_serving_write_leases", "community_deletion_executor_heartbeats", ] { diff --git a/crates/buzz-deletion/Cargo.toml b/crates/buzz-deletion/Cargo.toml index fb94cea13f..8c308b0a8c 100644 --- a/crates/buzz-deletion/Cargo.toml +++ b/crates/buzz-deletion/Cargo.toml @@ -13,6 +13,7 @@ thiserror = { workspace = true } buzz-core = { workspace = true } buzz-db = { workspace = true } buzz-media = { workspace = true } +chrono = { workspace = true } clap = { version = "4", features = ["derive"] } deadpool-redis = { workspace = true } hex = { workspace = true } diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 43171c6642..e67ef7037a 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -9,20 +9,31 @@ use std::time::Duration; use anyhow::{Context, Result}; use buzz_db::deletion::{ - ClaimedDeletion, DeletionRequest, DeletionStage, DeletionStore, FrozenInventory, LeaseToken, - StorageManifest, StorageObject, DEFAULT_LEASE_DURATION, + ClaimedDeletion, DeletionRequest, DeletionStage, DeletionStore, FrozenInventory, + KeyStreamDigest, LeaseToken, PrefixManifest, StorageManifest, TaxonomySweep, + DEFAULT_LEASE_DURATION, }; use buzz_db::{Db, DbConfig}; -use buzz_media::{deletion_inventory, CurrentObjectVersion, MediaStorage}; +use buzz_media::{is_tenant_owned_key, tenant_prefixes, MediaStorage}; use clap::Subcommand; use serde::Serialize; use tokio_util::sync::CancellationToken; use uuid::Uuid; -const DEFAULT_STORAGE_OBJECT_CAP: u64 = 1_000_000; +/// Per-community object cap; the fleet cap moved to the taxonomy sweep. +const DEFAULT_COMMUNITY_OBJECT_CAP: u64 = 1_000_000; +/// Fleet-wide object cap for one taxonomy sweep. +const DEFAULT_SWEEP_OBJECT_CAP: u64 = 10_000_000; +/// A deletion may rely on a clean sweep at most this old. +const DEFAULT_SWEEP_MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60); +/// Keys per frozen side-table chunk (one `DeleteObjects`-sized unit × 10). +const DEFAULT_MANIFEST_CHUNK_KEYS: usize = 10_000; +/// Unknown keys retained verbatim on a sweep record for diagnosis. +const SWEEP_UNKNOWN_KEY_SAMPLE: usize = 100; +/// One S3 LIST page. +const LIST_PAGE_SIZE: usize = 1000; const RETRY_DELAY: Duration = Duration::from_secs(30); const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10); -const STORAGE_DELETE_BATCH_SIZE: usize = 100; #[cfg(test)] static TEST_HEARTBEAT_INTERVAL_MS: AtomicU64 = AtomicU64::new(0); @@ -260,6 +271,12 @@ pub enum Command { #[arg(long)] executor_id: Option, }, + /// Sweep the whole bucket's key taxonomy and record the fleet evidence. + /// + /// Submit and fencing require a recent clean sweep instead of re-listing + /// the fleet bucket per deletion. Exits non-zero when unknown writer + /// shapes are found. + Sweep, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -346,11 +363,12 @@ pub async fn run(command: Command) -> Result { if host.is_empty() { anyhow::bail!("cannot derive community host; pass --host or set RELAY_URL"); } + let sweep = require_fresh_clean_sweep(&services).await?; let request = services .store .submit(&host, &requested_by, reason.as_deref()) .await?; - let inventory = build_inventory(&services, &request).await?; + let inventory = build_inventory(&services, &request, sweep.id).await?; let request = services .store .freeze_inventory(request.id, &inventory) @@ -397,6 +415,29 @@ pub async fn run(command: Command) -> Result { ) .await } + Command::Sweep => { + let started_at = chrono::Utc::now(); + let cap = sweep_object_cap(); + let media = Arc::clone(&services.media); + let outcome = + buzz_media::sweep_bucket_taxonomy(cap, SWEEP_UNKNOWN_KEY_SAMPLE, move |token| { + let media = Arc::clone(&media); + async move { media.list_page(token, LIST_PAGE_SIZE).await } + }) + .await?; + let sweep = services + .store + .record_taxonomy_sweep( + started_at, + outcome.listed_objects, + outcome.unknown_object_count, + &outcome.unknown_key_sample, + cap, + ) + .await?; + print_json(&sweep)?; + Ok(i32::from(sweep.unknown_object_count > 0)) + } } } @@ -465,52 +506,37 @@ where .unwrap_or(default) } -fn storage_taxonomy_matches(approved: &StorageManifest, live: &StorageManifest) -> bool { - approved.version == live.version - && approved.unknown_keys == live.unknown_keys - && approved.unsupported_version_keys == live.unsupported_version_keys -} - -fn storage_object_matches( - expected: &StorageObject, - observed_size: u64, - observed_e_tag: Option<&str>, -) -> bool { - expected.size == observed_size && observed_e_tag == expected.e_tag.as_deref() -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StorageObjectAction { - AlreadyMissing, - Delete, -} - -fn storage_object_action( - expected: &StorageObject, - current: &CurrentObjectVersion, -) -> Result { - match current { - CurrentObjectVersion::Missing => Ok(StorageObjectAction::AlreadyMissing), - CurrentObjectVersion::Present { - version_id: Some(_), - .. - } => Err(permanent(format!( - "unsupported object version after fence: {}", - expected.key - ))), - CurrentObjectVersion::Present { - version_id: None, - size, - e_tag, - .. - } if storage_object_matches(expected, *size, e_tag.as_deref()) => { - Ok(StorageObjectAction::Delete) - } - CurrentObjectVersion::Present { .. } => Err(permanent(format!( - "fenced object binding changed before deletion: {}", - expected.key - ))), +/// Gate destructive progress on recent fleet-wide taxonomy evidence. +/// +/// A missing or stale sweep is transient — the operator runs the sweep +/// command and retries. A dirty sweep is the fail-closed unknown-key +/// invariant and blocks permanently. +async fn require_fresh_clean_sweep(services: &Services) -> Result { + let sweep = services + .store + .latest_taxonomy_sweep() + .await? + .ok_or_else(|| { + transient("no storage taxonomy sweep recorded; run the deletions sweep command") + })?; + let age = chrono::Utc::now().signed_duration_since(sweep.completed_at); + let max_age = chrono::Duration::from_std(sweep_max_age()).unwrap_or(chrono::Duration::MAX); + if age > max_age { + return Err(transient(format!( + "latest storage taxonomy sweep {} is stale ({}h old); run the deletions sweep command", + sweep.id, + age.num_hours() + ))); + } + if sweep.unknown_object_count > 0 { + return Err(permanent(format!( + "latest storage taxonomy sweep {} found {} unknown object-store key(s); sample: {}", + sweep.id, + sweep.unknown_object_count, + sweep.unknown_key_sample.join(",") + ))); } + Ok(sweep) } fn validate_frozen_inventory(request: &DeletionRequest) -> Result { @@ -535,16 +561,15 @@ fn validate_frozen_inventory(request: &DeletionRequest) -> Result Result<()> { buzz_db::deletion::validate_storage_manifest(manifest)?; - let classified = deletion_inventory( - *request.community_id.as_uuid(), - manifest - .tenant_objects - .iter() - .map(|object| (object.key.clone(), object.size)), - ); - if classified.tenant_keys() != manifest.tenant_keys || !classified.unknown_keys.is_empty() { + let expected = tenant_prefixes(*request.community_id.as_uuid()); + let actual = manifest + .prefixes + .iter() + .map(|prefix| prefix.prefix.as_str()) + .collect::>(); + if actual != expected.iter().map(String::as_str).collect::>() { return Err(permanent( - "storage manifest contains keys not owned by the deletion target", + "storage manifest prefixes are not the deletion target's tenant prefixes", )); } Ok(()) @@ -553,80 +578,156 @@ fn validate_storage_ownership(request: &DeletionRequest, manifest: &StorageManif async fn build_inventory( services: &Services, request: &DeletionRequest, + sweep_id: Uuid, ) -> Result { let schema = services .store .inventory_schema(request.community_id) .await?; - let storage = build_storage_manifest(services, request).await?; + let storage = enumerate_tenant_prefixes(services, request, sweep_id, None, None).await?; Ok(FrozenInventory { schema, storage }) } -async fn build_storage_manifest_from_objects( +/// Buffered writer for frozen key chunks during the destructive freeze. +struct ChunkSink<'a> { + token: &'a LeaseToken, + next_chunk_no: i64, + buffered: Vec, +} + +async fn flush_chunk(services: &Services, sink: &mut ChunkSink<'_>, prefix: &str) -> Result<()> { + if sink.buffered.is_empty() { + return Ok(()); + } + services + .store + .append_manifest_key_chunk(sink.token, sink.next_chunk_no, prefix, &sink.buffered) + .await?; + sink.next_chunk_no += 1; + sink.buffered.clear(); + Ok(()) +} + +/// Enumerate the target's three tenant prefixes into per-prefix summaries. +/// +/// Cost is O(tenant objects) regardless of fleet size; the fleet-level +/// unknown-key invariant is covered by the taxonomy sweep gate instead. +/// Memory stays bounded at one listing page plus one buffered chunk — the +/// full key list is never materialized. When `sink` is supplied, keys are +/// also persisted as side-table chunks (never spanning prefixes) for the +/// destructive freeze to bind against these digests. +async fn enumerate_tenant_prefixes( services: &Services, request: &DeletionRequest, - objects: Vec, + sweep_id: Uuid, + heartbeat_lost: Option<&CancellationToken>, + mut sink: Option<&mut ChunkSink<'_>>, ) -> Result { - let bucket = deletion_inventory( - *request.community_id.as_uuid(), - objects - .iter() - .map(|object| (object.key.clone(), object.size)), - ); - let tenant_keys = bucket.tenant_keys(); - let tenant_key_set = tenant_keys.iter().collect::>(); - let tenant_objects = objects - .into_iter() - .filter(|object| tenant_key_set.contains(&object.key)) - .map(|object| StorageObject { - key: object.key, - size: object.size, - e_tag: object.e_tag, - }) - .collect::>(); - if tenant_objects.iter().any(|object| object.e_tag.is_none()) { + if services.media.bucket_versioning_detected().await? { return Err(permanent( - "object store omitted ETag for a tenant binding; conditional deletion is unavailable", + "bucket versioning detected; deletion cannot prove logical absence with delete markers", )); } - let mut tenant_objects = tenant_objects; - tenant_objects.sort(); - - let mut unsupported_version_keys = Vec::new(); - for key in &tenant_keys { - if matches!( - services.media.inspect_current_version(key).await?, - CurrentObjectVersion::Present { - version_id: Some(_), - .. + let community = *request.community_id.as_uuid(); + let cap = storage_object_cap(); + let chunk_keys = manifest_chunk_keys(); + let mut prefixes = Vec::new(); + let mut total_objects: u64 = 0; + for prefix in tenant_prefixes(community) { + let mut digest = KeyStreamDigest::new(); + let mut total_bytes: u64 = 0; + let mut continuation = None; + loop { + if heartbeat_lost.is_some_and(CancellationToken::is_cancelled) { + return Err(DeletionLeaseLost.into()); + } + let page = services + .media + .list_prefix_page(&prefix, continuation.take(), LIST_PAGE_SIZE) + .await?; + total_objects = total_objects.saturating_add(page.objects.len() as u64); + if total_objects > cap { + return Err(permanent(format!( + "community storage inventory exceeds per-community object cap {cap}" + ))); + } + for (key, size) in page.objects { + if !is_tenant_owned_key(community, &key) { + return Err(permanent(format!( + "key under a tenant prefix is outside the exact writer taxonomy: {key}" + ))); + } + digest.fold(&key)?; + total_bytes = total_bytes.saturating_add(size); + if let Some(sink) = sink.as_deref_mut() { + sink.buffered.push(key); + if sink.buffered.len() >= chunk_keys { + flush_chunk(services, sink, &prefix).await?; + } + } + } + if !page.is_truncated { + break; + } + continuation = page.next_continuation_token; + if continuation.is_none() { + return Err(transient( + "truncated tenant listing page has no continuation token", + )); } - ) { - unsupported_version_keys.push(key.clone()); } + if let Some(sink) = sink.as_deref_mut() { + flush_chunk(services, sink, &prefix).await?; + } + let (keys_digest, object_count) = digest.finish(); + prefixes.push(PrefixManifest { + prefix, + object_count, + total_bytes, + keys_digest, + }); } - let storage = StorageManifest { - version: 2, - tenant_keys, - tenant_objects, - git_pointer_keys: bucket.git_pointer_keys, - media_sidecar_keys: bucket.media_sidecar_keys, - media_upload_keys: bucket.media_upload_keys, - unknown_keys: bucket.unknown_keys, - unsupported_version_keys, + let manifest = StorageManifest { + version: 3, + prefixes, + taxonomy_sweep_id: sweep_id, + unknown_keys: Vec::new(), }; - buzz_db::deletion::validate_storage_manifest(&storage)?; - Ok(storage) + buzz_db::deletion::validate_storage_manifest(&manifest)?; + Ok(manifest) } -async fn build_storage_manifest( +/// Freeze the post-fence, post-drain destructive enumeration: stream the +/// tenant prefixes into side-table chunks, then bind the chunk stream to the +/// request row's digests atomically. +async fn freeze_destructive_manifest( services: &Services, request: &DeletionRequest, + token: &LeaseToken, + heartbeat_lost: &CancellationToken, ) -> Result { - let objects = services - .media - .list_all_for_deletion(storage_object_cap()) + let frozen = validate_frozen_inventory(request)?; + // A prior interrupted freeze may have left partial chunks; they were + // never bound to a committed manifest, so rewrite them from scratch. + services.store.clear_manifest_key_chunks(token).await?; + let mut sink = ChunkSink { + token, + next_chunk_no: 0, + buffered: Vec::new(), + }; + let manifest = enumerate_tenant_prefixes( + services, + request, + frozen.storage.taxonomy_sweep_id, + Some(heartbeat_lost), + Some(&mut sink), + ) + .await?; + services + .store + .freeze_destructive_storage_manifest(token, &manifest) .await?; - build_storage_manifest_from_objects(services, request, objects).await + Ok(manifest) } async fn run_loop( @@ -848,16 +949,16 @@ async fn run_stage_with_heartbeat( } } -async fn run_guarded_external_step( +async fn run_guarded_external_step( services: &Services, token: &LeaseToken, stage: DeletionStage, heartbeat_lost: &CancellationToken, operation: F, -) -> Result<()> +) -> Result where F: FnOnce() -> Fut, - Fut: std::future::Future>, + Fut: std::future::Future>, { services.store.verify_execution_token(token, stage).await?; let result = tokio::select! { @@ -867,9 +968,9 @@ where } result = operation() => result, }; - result?; + let output = result?; services.store.verify_execution_token(token, stage).await?; - Ok(()) + Ok(output) } async fn execute_stage( @@ -906,12 +1007,9 @@ async fn execute_stage( "approved structural catalog drifted before fencing", )); } - let live_storage = build_storage_manifest(services, request).await?; - if !storage_taxonomy_matches(&frozen.storage, &live_storage) { - return Err(permanent( - "approved storage taxonomy drifted before fencing", - )); - } + // The fleet unknown-key invariant is sweep evidence, not an + // inline re-listing of the whole bucket. + require_fresh_clean_sweep(services).await?; services.store.begin_quiescing(&token).await?; match services.store.fence(&token).await { Ok(_) => {} @@ -942,27 +1040,25 @@ async fn execute_stage( .store .verify_execution_token(&token, DeletionStage::Fenced) .await?; - let destructive = match request.destructive_storage_manifest.clone() { - Some(value) => serde_json::from_value(value)?, - None => { - let manifest = build_storage_manifest(services, request).await?; - services - .store - .freeze_destructive_storage_manifest(&token, &manifest) - .await?; - manifest - } - }; - validate_storage_ownership(request, &destructive)?; - if services + if !services .store .serving_writes_drained(request.community_id) .await? { - services.store.mark_drained(&token).await?; - } else { return Err(transient("serving writes have not drained")); } + // Freeze the destructive enumeration only after the fence closed + // new writers AND every admitted serving write drained: the + // post-drain listing is the final storage state, so no tenant key + // can appear after the freeze. + let destructive = match request.destructive_storage_manifest.clone() { + Some(value) => serde_json::from_value(value)?, + None => { + freeze_destructive_manifest(services, request, &token, heartbeat_lost).await? + } + }; + validate_storage_ownership(request, &destructive)?; + services.store.mark_drained(&token).await?; } DeletionStage::Drained => { let storage: StorageManifest = serde_json::from_value( @@ -972,67 +1068,72 @@ async fn execute_stage( .context("request has no post-fence destructive storage manifest")?, )?; validate_storage_ownership(request, &storage)?; - let completed = services.store.completed_storage_object_keys(&token).await?; - let mut processed = 0usize; - for object in storage - .tenant_objects - .iter() - .filter(|object| !completed.contains(&object.key)) - .take(storage_delete_batch_size()) - { - let current = services.media.inspect_current_object(&object.key).await?; - let already_missing = match storage_object_action(object, ¤t)? { - StorageObjectAction::AlreadyMissing => true, - StorageObjectAction::Delete => { - run_guarded_external_step( - services, - &token, - DeletionStage::Drained, - heartbeat_lost, - || async { - services.media.delete(&object.key).await?; - Ok(()) - }, - ) - .await?; - run_guarded_external_step( - services, - &token, - DeletionStage::Drained, - heartbeat_lost, - || async { - if services.media.head(&object.key).await? { - return Err(transient(format!( - "object binding still exists after delete: {}", - object.key - ))); - } - Ok(()) - }, - ) - .await?; - false - } - }; + // Resume = first unstamped chunk. Bulk deletes are idempotent + // (missing keys report as deleted), so re-deleting a chunk whose + // stamp was lost to a crash is safe. + let mut removed: u64 = 0; + let mut already_missing: u64 = 0; + while let Some(chunk) = services.store.next_pending_manifest_chunk(&token).await? { + let outcome = run_guarded_external_step( + services, + &token, + DeletionStage::Drained, + heartbeat_lost, + || async { Ok(services.media.delete_objects(&chunk.keys).await?) }, + ) + .await?; + if !outcome.versioned_keys.is_empty() { + return Err(permanent(format!( + "bulk delete produced version artifacts; bucket versioning blocks \ + deletion: {}", + outcome.versioned_keys.join(",") + ))); + } + if !outcome.failed.is_empty() { + let (key, code, message) = &outcome.failed[0]; + return Err(transient(format!( + "bulk delete failed for {} key(s); first: {key}: {code}: {message}", + outcome.failed.len() + ))); + } + let acknowledged = outcome.deleted.saturating_add(outcome.already_missing); + if acknowledged != chunk.keys.len() as u64 { + return Err(transient(format!( + "bulk delete acknowledged {acknowledged} of {} keys in chunk {}", + chunk.keys.len(), + chunk.chunk_no + ))); + } + removed += outcome.deleted; + already_missing += outcome.already_missing; services .store - .checkpoint_storage_object_removed(&token, &object.key, already_missing) + .mark_manifest_chunk_deleted( + &token, + chunk.chunk_no, + serde_json::json!({ + "prefix": chunk.prefix, + "keys": chunk.keys.len(), + "deleted": outcome.deleted, + "already_missing": outcome.already_missing, + }), + ) .await?; - processed += 1; - } - - let completed_count = completed.len().saturating_add(processed); - if completed_count < storage.tenant_objects.len() { - return Err(transient(format!( - "storage deletion batch complete: {completed_count}/{}", - storage.tenant_objects.len() - ))); } + let frozen_keys: u64 = storage + .prefixes + .iter() + .map(|prefix| prefix.object_count) + .sum(); services .store .mark_bindings_removed( &token, - serde_json::json!({"deleted_keys": storage.tenant_objects.len()}), + serde_json::json!({ + "deleted_keys": frozen_keys, + "removed_now": removed, + "already_missing": already_missing, + }), ) .await?; } @@ -1096,29 +1197,18 @@ fn token_with_current_fence(token: &LeaseToken, request: &DeletionRequest) -> Le } } -async fn verify_storage_absence_from_objects( - services: &Services, - request: &DeletionRequest, - objects: Vec, -) -> Result<()> { - let live = build_storage_manifest_from_objects(services, request, objects).await?; - if live.tenant_keys.is_empty() { - Ok(()) - } else { - Err(transient(format!( - "logical verification found {} live target object binding(s); first={}", - live.tenant_keys.len(), - live.tenant_keys.first().map_or("", String::as_str) - ))) - } -} - +/// Prove logical absence by listing each tenant prefix and requiring it +/// empty — O(1) requests per prefix, independent of fleet size. async fn verify_storage_absence(services: &Services, request: &DeletionRequest) -> Result<()> { - let objects = services - .media - .list_all_for_deletion(storage_object_cap()) - .await?; - verify_storage_absence_from_objects(services, request, objects).await + for prefix in tenant_prefixes(*request.community_id.as_uuid()) { + let page = services.media.list_prefix_page(&prefix, None, 1).await?; + if let Some((key, _)) = page.objects.first() { + return Err(transient(format!( + "logical verification found a live target object binding: {key}" + ))); + } + } + Ok(()) } async fn publish_disconnect_community( @@ -1219,16 +1309,33 @@ fn storage_object_cap() -> u64 { .ok() .and_then(|value| value.parse().ok()) .filter(|value| *value > 0) - .unwrap_or(DEFAULT_STORAGE_OBJECT_CAP) + .unwrap_or(DEFAULT_COMMUNITY_OBJECT_CAP) +} + +fn sweep_object_cap() -> u64 { + std::env::var("BUZZ_DELETION_SWEEP_MAX_OBJECTS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_SWEEP_OBJECT_CAP) } -fn storage_delete_batch_size() -> usize { - std::env::var("BUZZ_DELETION_STORAGE_DELETE_BATCH_SIZE") +fn sweep_max_age() -> Duration { + std::env::var("BUZZ_DELETION_SWEEP_MAX_AGE_SECS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value| *value > 0) + .map(Duration::from_secs) + .unwrap_or(DEFAULT_SWEEP_MAX_AGE) +} + +fn manifest_chunk_keys() -> usize { + std::env::var("BUZZ_DELETION_MANIFEST_CHUNK_KEYS") .ok() .and_then(|value| value.parse::().ok()) .filter(|value| *value > 0) - .map(|value| value.min(10_000)) - .unwrap_or(STORAGE_DELETE_BATCH_SIZE) + .map(|value| value.min(100_000)) + .unwrap_or(DEFAULT_MANIFEST_CHUNK_KEYS) } fn default_executor_id() -> String { @@ -1278,6 +1385,26 @@ fn print_json(value: &impl Serialize) -> Result<()> { mod tests { use super::*; + fn empty_storage_manifest( + community: buzz_core::CommunityId, + sweep_id: Uuid, + ) -> StorageManifest { + StorageManifest { + version: 3, + prefixes: tenant_prefixes(*community.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + taxonomy_sweep_id: sweep_id, + unknown_keys: Vec::new(), + } + } + async fn claimed_test_deletion(prefix: &str) -> (Services, ClaimedDeletion) { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) @@ -1297,21 +1424,16 @@ mod tests { .submit(&host, "test", None) .await .expect("submit deletion request"); + let sweep = store + .record_taxonomy_sweep(chrono::Utc::now(), 0, 0, &[], 1_000_000) + .await + .expect("record clean taxonomy sweep"); let inventory = FrozenInventory { schema: store .inventory_schema(community.id) .await .expect("inventory schema"), - storage: StorageManifest { - version: 2, - tenant_keys: Vec::new(), - tenant_objects: Vec::new(), - git_pointer_keys: Vec::new(), - media_sidecar_keys: Vec::new(), - media_upload_keys: Vec::new(), - unknown_keys: Vec::new(), - unsupported_version_keys: Vec::new(), - }, + storage: empty_storage_manifest(community.id, sweep.id), }; store .freeze_inventory(request.id, &inventory) @@ -1390,13 +1512,6 @@ mod tests { ) } - #[test] - fn large_storage_work_is_bounded_per_attempt() { - assert_eq!(STORAGE_DELETE_BATCH_SIZE, 100); - let keys = (0..1_000_000).take(STORAGE_DELETE_BATCH_SIZE).count(); - assert_eq!(keys, STORAGE_DELETE_BATCH_SIZE); - } - #[tokio::test] #[ignore = "requires Postgres"] async fn frozen_inventory_digest_and_storage_ownership_fail_closed() { @@ -1406,68 +1521,42 @@ mod tests { let mut digest_tampered = claim.request.clone(); digest_tampered.inventory_manifest = Some(serde_json::json!({ "schema": {"revision": 0, "migration_version": 0, "scoped_tables": [], "row_counts": {}, "fenced_tables": []}, - "storage": {"version": 2, "tenant_keys": [], "tenant_objects": [], "git_pointer_keys": [], "media_sidecar_keys": [], "media_upload_keys": [], "unknown_keys": [], "unsupported_version_keys": []} + "storage": {"version": 3, "prefixes": [], "taxonomy_sweep_id": Uuid::from_u128(1), "unknown_keys": []} })); assert!(validate_frozen_inventory(&digest_tampered).is_err()); - let foreign_community = Uuid::new_v4(); - let foreign_key = format!("_meta/{foreign_community}/{}.json", "a".repeat(64)); - let foreign_manifest = StorageManifest { - version: 2, - tenant_keys: vec![foreign_key.clone()], - tenant_objects: vec![StorageObject { - key: foreign_key, - size: 1, - e_tag: Some("etag".to_string()), - }], - git_pointer_keys: Vec::new(), - media_sidecar_keys: Vec::new(), - media_upload_keys: Vec::new(), - unknown_keys: Vec::new(), - unsupported_version_keys: Vec::new(), - }; + // A manifest scoped to another community's prefixes is never the + // deletion target's, even when internally valid. + let foreign_manifest = empty_storage_manifest( + buzz_core::CommunityId::from_uuid(Uuid::new_v4()), + Uuid::from_u128(1), + ); assert!(validate_storage_ownership(&claim.request, &foreign_manifest).is_err()); } + /// The non-atomic boundary under test: S3 committed a chunk's deletes, + /// then the worker died before the chunk stamp. Resume must re-delete the + /// chunk (missing keys report as deleted — idempotent), stamp it, and + /// finish the stage. #[tokio::test] #[ignore = "requires Postgres and S3-compatible storage"] - async fn drained_stage_reconciles_first_object_deleted_before_checkpoint() { - let (mut services, claim) = claimed_test_deletion("deletion-first-missing").await; - let media = deletion_test_media_storage(); - services.media = media; - - let object_key = format!( - "_meta/{}/{}.json", - claim.request.community_id, - "a".repeat(64) - ); - services - .media - .put(&object_key, b"crash-window", "application/json") - .await - .expect("seed object"); - let current = services - .media - .inspect_current_object(&object_key) - .await - .expect("inspect seeded object"); - let CurrentObjectVersion::Present { size, e_tag, .. } = current else { - panic!("seeded object must be present"); - }; - let storage = StorageManifest { - version: 2, - tenant_keys: vec![object_key.clone()], - tenant_objects: vec![StorageObject { - key: object_key.clone(), - size, - e_tag, - }], - git_pointer_keys: Vec::new(), - media_sidecar_keys: vec![object_key.clone()], - media_upload_keys: Vec::new(), - unknown_keys: Vec::new(), - unsupported_version_keys: Vec::new(), - }; + async fn drained_stage_resumes_chunk_deleted_before_stamp() { + let (mut services, claim) = claimed_test_deletion("deletion-chunk-resume").await; + services.media = deletion_test_media_storage(); + let community = claim.request.community_id; + let meta_prefix = format!("_meta/{community}/"); + let keys = vec![ + format!("{meta_prefix}{}.json", "a".repeat(64)), + format!("{meta_prefix}{}.json", "b".repeat(64)), + ]; + for key in &keys { + services + .media + .put(key, b"chunk-resume", "application/json") + .await + .expect("seed object"); + } + services .store .begin_quiescing(&claim.lease) @@ -1478,26 +1567,47 @@ mod tests { fence_generation: Some(generation), ..claim.lease.clone() }; + // Two single-key chunks so resume order is observable. + services + .store + .append_manifest_key_chunk(&token, 0, &meta_prefix, &keys[..1]) + .await + .expect("append chunk 0"); + services + .store + .append_manifest_key_chunk(&token, 1, &meta_prefix, &keys[1..]) + .await + .expect("append chunk 1"); + let mut digest = KeyStreamDigest::new(); + for key in &keys { + digest.fold(key).expect("fold key"); + } + let (keys_digest, object_count) = digest.finish(); + let mut storage = empty_storage_manifest(community, Uuid::from_u128(1)); + storage.taxonomy_sweep_id = validate_frozen_inventory(&claim.request) + .expect("frozen inventory") + .storage + .taxonomy_sweep_id; + storage.prefixes[0] = PrefixManifest { + prefix: meta_prefix.clone(), + object_count, + total_bytes: keys.len() as u64 * "chunk-resume".len() as u64, + keys_digest, + }; services .store .freeze_destructive_storage_manifest(&token, &storage) .await - .expect("freeze one-object manifest"); + .expect("freeze chunked manifest"); services.store.mark_drained(&token).await.expect("drained"); - // This is the non-atomic boundary under test: S3 committed the delete, - // then the worker died before checkpoint_storage_object_removed. + // Simulate the crash window: chunk 0's key is already gone from S3 + // but the chunk was never stamped. services .media - .delete(&object_key) - .await - .expect("simulate committed delete before crash"); - assert!(services - .store - .completed_storage_object_keys(&token) + .delete(&keys[0]) .await - .expect("initial checkpoints") - .is_empty()); + .expect("simulate committed delete before stamp"); let resumed = ClaimedDeletion { request: services @@ -1509,32 +1619,26 @@ mod tests { }; execute_stage(&services, &resumed, &CancellationToken::new()) .await - .expect("production Drained stage reconciles missing first object"); + .expect("Drained stage resumes at the unstamped chunk"); assert_eq!( services .store - .completed_storage_object_keys(&token) + .manifest_chunk_progress(token.request_id) .await - .expect("completed checkpoints"), - std::collections::BTreeSet::from([object_key]) + .expect("chunk progress"), + (2, 2) ); assert_eq!( services.store.get(token.request_id).await.unwrap().stage, DeletionStage::BindingsRemoved ); - } - - #[test] - fn object_identity_rejects_replacement_but_accepts_exact_match() { - let expected = StorageObject { - key: "tenant/key".to_string(), - size: 42, - e_tag: Some("etag-a".to_string()), - }; - assert!(storage_object_matches(&expected, 42, Some("etag-a"))); - assert!(!storage_object_matches(&expected, 42, Some("etag-b"))); - assert!(!storage_object_matches(&expected, 42, None)); + for key in &keys { + assert!( + !services.media.head(key).await.expect("verify absence"), + "tenant binding {key} must be gone" + ); + } } #[test] @@ -1578,24 +1682,29 @@ mod tests { } #[tokio::test] - #[ignore = "requires Postgres"] + #[ignore = "requires Postgres and S3-compatible storage"] async fn final_storage_verification_rejects_late_target_binding() { - let (services, claim) = claimed_test_deletion("deletion-late-binding").await; + let (mut services, claim) = claimed_test_deletion("deletion-late-binding").await; + services.media = deletion_test_media_storage(); let community = claim.request.community_id; let late_key = format!("_meta/{community}/{}.json", "a".repeat(64)); - let error = verify_storage_absence_from_objects( - &services, - &claim.request, - vec![buzz_media::DeletionObject { - key: late_key.clone(), - size: 7, - last_modified: "2026-08-02T00:00:00Z".to_string(), - e_tag: Some("etag".to_string()), - }], - ) - .await - .expect_err("late target binding must fail verification"); + services + .media + .put(&late_key, b"late", "application/json") + .await + .expect("seed late binding"); + let error = verify_storage_absence(&services, &claim.request) + .await + .expect_err("late target binding must fail verification"); assert!(format!("{error:#}").contains(&late_key)); + services + .media + .delete(&late_key) + .await + .expect("remove late binding"); + verify_storage_absence(&services, &claim.request) + .await + .expect("empty tenant prefixes verify clean"); } #[tokio::test] diff --git a/crates/buzz-media/src/bucket_index.rs b/crates/buzz-media/src/bucket_index.rs index d05656cba2..6c78c2e0a1 100644 --- a/crates/buzz-media/src/bucket_index.rs +++ b/crates/buzz-media/src/bucket_index.rs @@ -754,51 +754,102 @@ mod tests { } } -/// Classify a full bucket listing for whole-community deletion. +/// The exact community-scoped listing prefixes owned by one tenant, in +/// ascending key order: media sidecars, upload records, and Git repository +/// pointers. Every tenant-owned binding lives under one of these; shared +/// immutable CAS/thumb/probe data is deliberately outside them (fleet-wide +/// physical GC is a separate retention phase). +pub fn tenant_prefixes(community: Uuid) -> [String; 3] { + [ + format!("_meta/{community}/"), + format!("_uploads/{community}/"), + format!("repos/{community}/"), + ] +} + +/// Whether one bucket key is a tenant-owned binding of `community` in the +/// exact writer taxonomy: a media sidecar, an upload record, or a Git +/// repository pointer. A malformed key under a tenant prefix is NOT owned — +/// deletion fails closed on shapes this binary did not write. +pub fn is_tenant_owned_key(community: Uuid, key: &str) -> bool { + match classify_key(key) { + KeyClass::Sidecar { + community: owner, .. + } + | KeyClass::Auxiliary { + community: owner, .. + } => owner == community, + KeyClass::Unknown => git_pointer_community(key) == Some(community), + KeyClass::Blob { .. } | KeyClass::Thumb { .. } => false, + } +} + +/// Whether one bucket key belongs to the fleet's known writer taxonomy: +/// blob/thumb/sidecar/upload shapes, any community's Git pointer, shared Git +/// CAS data, or a `probe/` connectivity key. +pub fn is_known_fleet_key(key: &str) -> bool { + !matches!(classify_key(key), KeyClass::Unknown) + || git_pointer_community(key).is_some() + || is_known_git_shared_key(key) + || key.starts_with("probe/") +} + +/// Durable outcome of one fleet-wide taxonomy sweep. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct TaxonomySweepOutcome { + /// Total objects listed. + pub listed_objects: u64, + /// Exact count of keys outside the known writer taxonomy. + pub unknown_object_count: u64, + /// Bounded sample of unknown keys, in listing order. + pub unknown_key_sample: Vec, +} + +/// Fold an entire paginated bucket listing into the fleet taxonomy outcome. /// -/// Unknown keys are reported globally rather than ignored: deleting a tenant -/// while the bucket contains a writer shape this binary does not understand is -/// unsafe. Tenant-owned sidecars/upload records are returned separately from -/// shared immutable blob/thumb and Git CAS data. -pub fn deletion_inventory( - community: Uuid, - objects: impl IntoIterator, -) -> DeletionBucketInventory { - let mut inventory = DeletionBucketInventory::default(); - let repo_prefix = format!("repos/{community}/"); - for (key, _size) in objects { - match classify_key(&key) { - KeyClass::Sidecar { - community: owner, .. - } if owner == community => inventory.media_sidecar_keys.push(key), - KeyClass::Auxiliary { - community: owner, .. - } if owner == community => inventory.media_upload_keys.push(key), - KeyClass::Sidecar { .. } - | KeyClass::Auxiliary { .. } - | KeyClass::Blob { .. } - | KeyClass::Thumb { .. } => {} - KeyClass::Unknown => { - if let Some(owner) = git_pointer_community(&key) { - if owner == community && key.starts_with(&repo_prefix) { - inventory.git_pointer_keys.push(key); - } - } else if is_known_git_shared_key(&key) || key.starts_with("probe/") { - // Known shared immutable CAS/probe data is deliberately - // outside the per-community frozen manifest. V1 removes - // only tenant-owned bindings; fleet-wide physical GC is a - // separate retention phase. - } else { - inventory.unknown_keys.push(key); +/// Deleting a tenant while the bucket contains a writer shape this binary +/// does not understand is unsafe — but that is a *fleet* invariant, not a +/// per-request one. This sweep records it once; deletion stages then gate on +/// a recent clean sweep instead of re-listing the whole bucket per request. +/// Same pagination/cap contract as [`fold_bucket_listing`]; memory is +/// bounded by `sample_limit`, never the listing size. +pub async fn sweep_bucket_taxonomy( + cap: u64, + sample_limit: usize, + mut fetch_page: F, +) -> Result +where + F: FnMut(Option) -> Fut, + Fut: Future>, +{ + let mut outcome = TaxonomySweepOutcome::default(); + let mut continuation_token = None; + loop { + let page = fetch_page(continuation_token.take()).await?; + outcome.listed_objects += page.objects.len() as u64; + if outcome.listed_objects > cap { + return Err(SweepError::CapExceeded { + seen: outcome.listed_objects, + cap, + }); + } + for (key, _size) in page.objects { + if !is_known_fleet_key(&key) { + outcome.unknown_object_count += 1; + if outcome.unknown_key_sample.len() < sample_limit { + outcome.unknown_key_sample.push(key); } } } + if !page.is_truncated { + break; + } + match page.next_continuation_token { + Some(token) => continuation_token = Some(token), + None => return Err(SweepError::MalformedPage), + } } - inventory.media_sidecar_keys.sort(); - inventory.media_upload_keys.sort(); - inventory.git_pointer_keys.sort(); - inventory.unknown_keys.sort(); - inventory + Ok(outcome) } fn git_pointer_community(key: &str) -> Option { @@ -835,64 +886,125 @@ fn is_known_git_shared_key(key: &str) -> bool { }) } -/// Object-store inventory relevant to one community deletion. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct DeletionBucketInventory { - /// Community media serve-gate sidecars. - pub media_sidecar_keys: Vec, - /// Community upload/moderation records. - pub media_upload_keys: Vec, - /// Community Git repository pointers. - pub git_pointer_keys: Vec, - /// Bucket keys outside the exact writer taxonomy. - pub unknown_keys: Vec, -} - -impl DeletionBucketInventory { - /// Sorted union of all tenant-owned binding keys. - pub fn tenant_keys(&self) -> Vec { - let mut keys = self.media_sidecar_keys.clone(); - keys.extend(self.media_upload_keys.iter().cloned()); - keys.extend(self.git_pointer_keys.iter().cloned()); - keys.sort(); - keys - } -} - #[cfg(test)] -mod deletion_inventory_tests { +mod deletion_taxonomy_tests { use super::*; #[test] - fn inventories_target_bindings_and_ignores_shared_cas() { + fn tenant_ownership_is_exact_per_community_and_shape() { let target = Uuid::from_u128(1); let other = Uuid::from_u128(2); let sha = "a".repeat(64); - let objects = vec![ - (format!("_meta/{target}/{sha}.json"), 1), - (format!("_meta/{other}/{sha}.json"), 1), + + assert!(is_tenant_owned_key( + target, + &format!("_meta/{target}/{sha}.json") + )); + assert!(is_tenant_owned_key( + target, + &format!("_uploads/{target}/{sha}/01ARZ3NDEKTSV4RRFFQ69G5FAV.json") + )); + assert!(is_tenant_owned_key( + target, + &format!("repos/{target}/{}/repo/pointer", "b".repeat(64)) + )); + // Another tenant's bindings and shared CAS are never owned. + assert!(!is_tenant_owned_key( + target, + &format!("_meta/{other}/{sha}.json") + )); + assert!(!is_tenant_owned_key(target, &format!("{sha}.png"))); + // A malformed key under the tenant's own prefix fails closed. + assert!(!is_tenant_owned_key( + target, + &format!("_meta/{target}/not-a-sidecar") + )); + assert!(!is_tenant_owned_key( + target, + &format!("repos/{target}/stray-file") + )); + } + + #[test] + fn tenant_prefixes_cover_every_owned_shape_and_sort_ascending() { + let community = Uuid::from_u128(7); + let prefixes = tenant_prefixes(community); + assert!(prefixes.windows(2).all(|pair| pair[0] < pair[1])); + let sha = "c".repeat(64); + for key in [ + format!("_meta/{community}/{sha}.json"), + format!("_uploads/{community}/{sha}/01ARZ3NDEKTSV4RRFFQ69G5FAV.json"), + format!("repos/{community}/{}/repo/pointer", "d".repeat(64)), + ] { + assert!( + prefixes + .iter() + .any(|prefix| key.starts_with(prefix.as_str())), + "owned key {key} must live under a tenant prefix" + ); + assert!(is_tenant_owned_key(community, &key)); + } + } + + #[tokio::test] + async fn taxonomy_sweep_counts_all_unknowns_but_bounds_the_sample() { + let community = Uuid::from_u128(1); + let sha = "a".repeat(64); + let known = vec![ + (format!("{sha}.png"), 1), + (format!("{sha}.thumb.jpg"), 1), + (format!("_meta/{community}/{sha}.json"), 1), + (format!("packs/{sha}"), 1), ( - format!("_uploads/{target}/{sha}/01ARZ3NDEKTSV4RRFFQ69G5FAV.json"), + format!("repos/{community}/{}/repo/pointer", "b".repeat(64)), 1, ), - (format!("repos/{target}/{}/repo/pointer", "b".repeat(64)), 1), - (format!("packs/{sha}"), 1), - (format!("{sha}.png"), 1), + ("probe/cas-123.txt".to_string(), 1), ]; - let inventory = deletion_inventory(target, objects); - assert_eq!(inventory.media_sidecar_keys.len(), 1); - assert_eq!(inventory.media_upload_keys.len(), 1); - assert_eq!(inventory.git_pointer_keys.len(), 1); - assert!(inventory.unknown_keys.is_empty()); - assert_eq!(inventory.tenant_keys().len(), 3); + let pages = [ + Page { + objects: known, + next_continuation_token: Some("next".to_string()), + is_truncated: true, + }, + Page { + objects: vec![ + ("future-format/one".to_string(), 1), + ("future-format/two".to_string(), 1), + ("future-format/three".to_string(), 1), + ], + next_continuation_token: None, + is_truncated: false, + }, + ]; + let outcome = sweep_bucket_taxonomy(100, 2, |token| { + let page = match token.as_deref() { + None => pages[0].clone(), + Some("next") => pages[1].clone(), + other => panic!("unexpected continuation token {other:?}"), + }; + async move { Ok(page) } + }) + .await + .expect("sweep synthetic listing"); + assert_eq!(outcome.listed_objects, 9); + assert_eq!(outcome.unknown_object_count, 3); + assert_eq!( + outcome.unknown_key_sample, + vec!["future-format/one", "future-format/two"] + ); } - #[test] - fn unknown_writer_shape_is_not_silently_skipped() { - let inventory = deletion_inventory( - Uuid::from_u128(1), - vec![("future-format/community/data".to_string(), 1)], - ); - assert_eq!(inventory.unknown_keys, vec!["future-format/community/data"]); + #[tokio::test] + async fn taxonomy_sweep_fails_closed_past_the_fleet_cap() { + let result = sweep_bucket_taxonomy(1, 10, |_token| async { + Ok(Page { + objects: vec![("a".to_string(), 1), ("b".to_string(), 1)], + next_continuation_token: None, + is_truncated: false, + }) + }) + .await; + assert!(matches!(result, Err(SweepError::CapExceeded { .. }))); } } diff --git a/crates/buzz-media/src/lib.rs b/crates/buzz-media/src/lib.rs index 0aeacfadde..b2ff12c16e 100644 --- a/crates/buzz-media/src/lib.rs +++ b/crates/buzz-media/src/lib.rs @@ -14,14 +14,13 @@ pub mod upload_record; pub mod validation; pub use bucket_index::{ - classify_key, deletion_inventory, fold_bucket_listing, BucketAggregate, BucketSnapshot, - CommunityStorage, DeletionBucketInventory, KeyClass, Page, SweepError, + classify_key, fold_bucket_listing, is_tenant_owned_key, sweep_bucket_taxonomy, tenant_prefixes, + BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass, Page, SweepError, + TaxonomySweepOutcome, }; pub use config::{MediaConfig, S3AddressingStyle}; pub use error::MediaError; -pub use storage::{ - BlobHeadMeta, BlobMeta, ByteStream, CurrentObjectVersion, DeletionObject, MediaStorage, -}; +pub use storage::{BlobHeadMeta, BlobMeta, BulkDeleteOutcome, ByteStream, MediaStorage}; pub use types::BlobDescriptor; pub use upload::{process_file_upload, process_upload, process_video_upload}; pub use upload_record::{ diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index f2a0ca7158..f9b642b5ae 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -177,81 +177,45 @@ impl MediaStorage { } } - /// Exhaustively list bucket objects for deletion inventory. + /// Detect whether the bucket has ever had versioning enabled. /// - /// Fails rather than returning a partial listing when pagination is - /// malformed or the explicit object cap is exceeded. - pub async fn list_all_for_deletion( - &self, - max_objects: u64, - ) -> Result, MediaError> { - let mut objects = Vec::new(); - let mut continuation = None; - loop { - let (result, _status) = self - .bucket - .list_page(String::new(), None, continuation.take(), None, Some(1000)) - .await?; - let next_len = objects.len().saturating_add(result.contents.len()); - if u64::try_from(next_len).unwrap_or(u64::MAX) > max_objects { - return Err(MediaError::StorageError(format!( - "deletion inventory exceeds object cap {max_objects}" - ))); - } - objects.extend(result.contents.into_iter().map(|object| DeletionObject { - key: object.key, - size: object.size, - last_modified: object.last_modified, - e_tag: object.e_tag, - })); - if !result.is_truncated { - break; - } - continuation = result.next_continuation_token; - if continuation.is_none() { - return Err(MediaError::StorageError( - "truncated deletion inventory page has no continuation token".to_string(), - )); - } - } - Ok(objects) - } - - /// Return the current object identity, including whether it is versioned. - pub async fn inspect_current_object( - &self, - key: &str, - ) -> Result { - match self.bucket.head_object(key).await { - Ok((result, _)) => Ok(CurrentObjectVersion::Present { - version_id: result.version_id, - size: result.content_length.unwrap_or_default().max(0) as u64, - last_modified: result.last_modified, - e_tag: result.e_tag, - }), - Err(s3::error::S3Error::HttpFailWithBody(404, _)) => Ok(CurrentObjectVersion::Missing), - Err(error) => Err(MediaError::StorageError(error.to_string())), - } + /// rust-s3 exposes no GetBucketVersioning, so this writes and inspects a + /// short-lived fleet probe object instead: versioning-enabled (and + /// versioning-suspended) buckets stamp new writes with a version id. + /// Deletion refuses versioned buckets because bulk deletes without a + /// VersionId would only insert delete markers, not prove logical absence. + pub async fn bucket_versioning_detected(&self) -> Result { + let key = format!("probe/deletion-versioning-{}", uuid::Uuid::new_v4()); + self.put(&key, b"buzz deletion versioning probe", "text/plain") + .await?; + let inspected = self.bucket.head_object(&key).await; + let removed = self.bucket.delete_object(&key).await; + let (head, _) = inspected.map_err(|e| MediaError::StorageError(e.to_string()))?; + removed.map_err(|e| MediaError::StorageError(e.to_string()))?; + Ok(head.version_id.is_some()) } - /// Return the current object version id, if the backend reports one. + /// Bulk-delete up to one manifest chunk of keys via S3 `DeleteObjects`. /// - /// V1 refuses to delete versioned tenant bindings because rust-s3 cannot - /// exhaustively enumerate non-current versions; a plain delete marker would - /// not prove logical absence from storage. - pub async fn inspect_current_version( - &self, - key: &str, - ) -> Result { - self.inspect_current_object(key).await - } - - /// Delete tenant binding keys one at a time and fail on any backend error. - pub async fn delete_bindings(&self, keys: &[String]) -> Result<(), MediaError> { - for key in keys { - self.delete(key).await?; + /// Never fails on per-key outcomes: they are folded into + /// [`BulkDeleteOutcome`] so the caller owns retry/fail-closed policy. + /// Historical MinIO releases report already-absent keys as + /// `NoSuchKey`/`NoSuchVersion` errors instead of deleted; both map to + /// `already_missing` to keep checkpointed retry idempotent. + pub async fn delete_objects(&self, keys: &[String]) -> Result { + if keys.is_empty() { + return Ok(BulkDeleteOutcome::default()); } - Ok(()) + let identifiers = keys + .iter() + .map(|key| s3::serde_types::ObjectIdentifier::new(key.clone())) + .collect::>(); + let result = self + .bucket + .delete_objects(identifiers) + .await + .map_err(|e| MediaError::StorageError(e.to_string()))?; + Ok(fold_bulk_delete_result(result)) } /// Build the community-scoped sidecar key for a given sha256 (bare hash). @@ -328,11 +292,28 @@ impl MediaStorage { &self, continuation_token: Option, max_keys: usize, + ) -> Result { + self.list_prefix_page("", continuation_token, max_keys) + .await + } + + /// One page of a prefix-scoped listing. + /// + /// Deletion enumerates the target community's exact key prefixes with + /// this instead of listing the whole fleet bucket: cost stays + /// O(tenant objects) regardless of fleet size. `ListObjectsV2` returns + /// keys in ascending UTF-8 binary order, which callers rely on for + /// streaming key-stream digests. + pub async fn list_prefix_page( + &self, + prefix: &str, + continuation_token: Option, + max_keys: usize, ) -> Result { let (result, _status) = self .bucket .list_page( - String::new(), + prefix.to_string(), None, continuation_token, None, @@ -351,35 +332,43 @@ impl MediaStorage { } } -/// Current-version observation for a tenant-owned object binding. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CurrentObjectVersion { - /// The key does not exist. - Missing, - /// The key exists, optionally with a backend version id. - Present { - /// S3 version id. `Some` is unsupported for V1 deletion. - version_id: Option, - /// Current object size. - size: u64, - /// Current last-modified timestamp, when supplied. - last_modified: Option, - /// Current entity tag, when supplied. - e_tag: Option, - }, +/// Per-key outcomes of one bulk `DeleteObjects` call. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct BulkDeleteOutcome { + /// Keys the backend reported deleted (S3 reports already-missing keys as + /// deleted too — the API is idempotent by design). + pub deleted: u64, + /// Keys reported absent via legacy MinIO `NoSuchKey`/`NoSuchVersion` + /// per-key errors; equivalent to deleted for retry purposes. + pub already_missing: u64, + /// Keys whose deletion produced a version artifact (delete marker or + /// version id) — evidence of bucket versioning, which deletion must + /// fail closed on. + pub versioned_keys: Vec, + /// Remaining per-key failures as `(key, code, message)`. + pub failed: Vec<(String, String, String)>, } -/// One object observed during exhaustive deletion inventory. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct DeletionObject { - /// Exact bucket key. - pub key: String, - /// Current object size. - pub size: u64, - /// Backend-reported last modification timestamp. - pub last_modified: String, - /// Backend entity tag, when supplied. - pub e_tag: Option, +fn fold_bulk_delete_result(result: s3::serde_types::DeleteObjectsResult) -> BulkDeleteOutcome { + let mut outcome = BulkDeleteOutcome::default(); + for deleted in result.deleted { + if deleted.delete_marker == Some(true) + || deleted.delete_marker_version_id.is_some() + || deleted.version_id.is_some() + { + outcome.versioned_keys.push(deleted.key); + } else { + outcome.deleted += 1; + } + } + for error in result.errors { + if error.code == "NoSuchKey" || error.code == "NoSuchVersion" { + outcome.already_missing += 1; + } else { + outcome.failed.push((error.key, error.code, error.message)); + } + } + outcome } #[cfg(test)] @@ -387,6 +376,49 @@ mod tests { use super::*; use std::collections::HashMap; + /// The bulk-delete fold is the retry-idempotence contract: legacy MinIO + /// absent-key errors count as success, version artifacts are surfaced for + /// fail-closed handling, and anything else stays a per-key failure. + #[test] + fn bulk_delete_fold_maps_absent_keys_and_version_artifacts() { + use s3::serde_types::{DeleteError, DeleteObjectsResult, DeletedObject}; + let deleted_object = |key: &str, marker: bool| DeletedObject { + key: key.to_string(), + version_id: None, + delete_marker: marker.then_some(true), + delete_marker_version_id: marker.then(|| "v1".to_string()), + }; + let delete_error = |key: &str, code: &str, message: &str| DeleteError { + key: key.to_string(), + code: code.to_string(), + message: message.to_string(), + version_id: None, + }; + let result = DeleteObjectsResult { + deleted: vec![ + deleted_object("plain", false), + deleted_object("marked", true), + ], + errors: vec![ + delete_error("gone", "NoSuchKey", "absent"), + delete_error("gone-version", "NoSuchVersion", "absent"), + delete_error("denied", "AccessDenied", "nope"), + ], + }; + let outcome = fold_bulk_delete_result(result); + assert_eq!(outcome.deleted, 1); + assert_eq!(outcome.already_missing, 2); + assert_eq!(outcome.versioned_keys, vec!["marked".to_string()]); + assert_eq!( + outcome.failed, + vec![( + "denied".to_string(), + "AccessDenied".to_string(), + "nope".to_string() + )] + ); + } + fn tenant(n: u128) -> TenantContext { TenantContext::resolved( CommunityId::from_uuid(uuid::Uuid::from_u128(n)), diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index f662e5b748..556abcad84 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -2326,27 +2326,38 @@ mod track_c_tests { buzz_db::deletion::DeletionRequest, buzz_db::deletion::ClaimedDeletion, ) { - use buzz_db::deletion::{FrozenInventory, StorageManifest, DEFAULT_LEASE_DURATION}; + use buzz_db::deletion::{ + FrozenInventory, KeyStreamDigest, PrefixManifest, StorageManifest, + DEFAULT_LEASE_DURATION, + }; let store = state.db.deletion_store(); let request = store .submit(host, "git-finalize-test", Some("post-CAS lease regression")) .await .expect("submit deletion"); + let sweep = store + .record_taxonomy_sweep(chrono::Utc::now(), 0, 0, &[], 1_000_000) + .await + .expect("record clean taxonomy sweep"); let inventory = FrozenInventory { schema: store .inventory_schema(request.community_id) .await .expect("schema inventory"), storage: StorageManifest { - version: 2, - tenant_keys: Vec::new(), - tenant_objects: Vec::new(), - git_pointer_keys: Vec::new(), - media_sidecar_keys: Vec::new(), - media_upload_keys: Vec::new(), + version: 3, + prefixes: buzz_media::tenant_prefixes(*request.community_id.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + taxonomy_sweep_id: sweep.id, unknown_keys: Vec::new(), - unsupported_version_keys: Vec::new(), }, }; store diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index 93caa6c0ae..947128b307 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -3164,7 +3164,10 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] async fn ingest_write_fence_follows_community_deletion_lifecycle() { - use buzz_db::deletion::{FrozenInventory, StorageManifest, DEFAULT_LEASE_DURATION}; + use buzz_db::deletion::{ + FrozenInventory, KeyStreamDigest, PrefixManifest, StorageManifest, + DEFAULT_LEASE_DURATION, + }; let url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) @@ -3194,20 +3197,28 @@ mod tests { ) .await .expect("submit"); + let sweep = store + .record_taxonomy_sweep(chrono::Utc::now(), 0, 0, &[], 1_000_000) + .await + .expect("record clean taxonomy sweep"); let inventory = FrozenInventory { schema: store .inventory_schema(community) .await .expect("schema inventory"), storage: StorageManifest { - version: 2, - tenant_keys: Vec::new(), - tenant_objects: Vec::new(), - git_pointer_keys: Vec::new(), - media_sidecar_keys: Vec::new(), - media_upload_keys: Vec::new(), + version: 3, + prefixes: buzz_media::tenant_prefixes(*community.as_uuid()) + .into_iter() + .map(|prefix| PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: KeyStreamDigest::new().finish().0, + }) + .collect(), + taxonomy_sweep_id: sweep.id, unknown_keys: Vec::new(), - unsupported_version_keys: Vec::new(), }, }; let request = store diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 810a8210cb..79ffd930eb 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -146,6 +146,82 @@ CREATE TABLE community_deletion_checkpoints ( CHECK ((status = 'failed') = (error IS NOT NULL)) ); +-- Frozen destructive key list, chunked out of the request row so a large +-- tenant (100k-1M objects) never materializes as one multi-hundred-MB JSONB +-- value. Rows are written once in the fenced stage, stamped `deleted_at` as +-- the executor confirms each chunk removed, and dropped at logical +-- verification. The request row keeps only per-prefix count/bytes/digest +-- summaries; the chunk stream must hash to those frozen digests. +CREATE TABLE community_deletion_manifest_keys ( + request_id UUID NOT NULL REFERENCES community_deletion_requests(id) ON DELETE CASCADE, + chunk_no BIGINT NOT NULL CHECK (chunk_no >= 0), + prefix TEXT NOT NULL, + keys JSONB NOT NULL, + deleted_at TIMESTAMPTZ, + PRIMARY KEY (request_id, chunk_no) +); + +-- Chunk content is immutable once written; the only permitted update is the +-- one-way deleted_at stamp. Removal is permitted only while the destructive +-- manifest has not yet frozen (a retried partial freeze rewrites its chunks) +-- or once the request has passed logical verification (terminal cleanup). +CREATE FUNCTION protect_community_deletion_manifest_keys() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + frozen_at TIMESTAMPTZ; + request_stage TEXT; +BEGIN + IF TG_OP = 'UPDATE' THEN + IF NEW.request_id IS DISTINCT FROM OLD.request_id + OR NEW.chunk_no IS DISTINCT FROM OLD.chunk_no + OR NEW.prefix IS DISTINCT FROM OLD.prefix + OR NEW.keys IS DISTINCT FROM OLD.keys + OR OLD.deleted_at IS NOT NULL + THEN + RAISE EXCEPTION 'community deletion manifest key chunks are immutable' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; + RETURN NEW; + END IF; + SELECT destructive_storage_frozen_at, stage + INTO frozen_at, request_stage + FROM community_deletion_requests + WHERE id = OLD.request_id; + IF NOT FOUND + OR frozen_at IS NULL + OR request_stage IN ('logically_verified', 'retention_pending') + THEN + RETURN OLD; + END IF; + RAISE EXCEPTION 'community deletion manifest key chunks cannot be removed mid-execution' + USING ERRCODE = 'integrity_constraint_violation'; +END; +$$; + +CREATE TRIGGER community_deletion_manifest_keys_guard +BEFORE UPDATE OR DELETE ON community_deletion_manifest_keys +FOR EACH ROW +EXECUTE FUNCTION protect_community_deletion_manifest_keys(); + +-- Fleet-wide object-store taxonomy sweep evidence. The unknown-key +-- fail-closed invariant is a fleet property, not a per-request one: deletion +-- inventory and fencing require a recent clean sweep instead of re-listing +-- the whole bucket inline per request. +CREATE TABLE storage_taxonomy_sweeps ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + started_at TIMESTAMPTZ NOT NULL, + completed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + listed_objects BIGINT NOT NULL CHECK (listed_objects >= 0), + unknown_object_count BIGINT NOT NULL CHECK (unknown_object_count >= 0), + unknown_key_sample JSONB NOT NULL DEFAULT '[]'::jsonb, + object_cap BIGINT NOT NULL CHECK (object_cap > 0), + CHECK (completed_at >= started_at) +); +CREATE INDEX storage_taxonomy_sweeps_latest + ON storage_taxonomy_sweeps (completed_at DESC); + CREATE TABLE community_serving_write_leases ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), community_id UUID NOT NULL REFERENCES communities(id), @@ -175,6 +251,8 @@ INSERT INTO _operator_global_tables (table_name, reason) VALUES ('community_deletion_requests', 'deployment deletion lifecycle and frozen inventory'), ('community_deletion_approvals', 'deployment operator destructive approvals'), ('community_deletion_checkpoints', 'deployment deletion executor checkpoints and failures'), + ('community_deletion_manifest_keys', 'deployment deletion frozen destructive key chunks'), + ('storage_taxonomy_sweeps', 'deployment object-store taxonomy sweep evidence'), ('community_serving_write_leases', 'deployment serving side-effect leases drained by deletion'), ('community_deletion_executor_heartbeats', 'deployment deletion worker liveness'); From a83c51cab6fb889252ab57a893d6a1cf8a93a75c Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Mon, 3 Aug 2026 13:21:27 -0700 Subject: [PATCH 23/51] fix: close community deletion safety races Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 68 +++++++++++++++++++++++--- crates/buzz-deletion/src/lib.rs | 37 ++++++++++++-- migrations/0028_community_deletion.sql | 20 ++++++-- 3 files changed, 111 insertions(+), 14 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 1ea3eac27b..31da75fac2 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -1234,6 +1234,18 @@ impl DeletionStore { ) -> Result<()> { let generation = require_fence_generation(token)?; let mut tx = self.pool.begin().await?; + // Serialize the freeze boundary with chunk INSERTs. The database trigger + // takes the same request-row lock before admitting each new chunk. + sqlx::query("SELECT id FROM community_deletion_requests WHERE id = $1 FOR UPDATE") + .bind(token.request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion request {} disappeared before manifest freeze", + token.request_id + )) + })?; verify_lease_and_fence(&mut tx, token, DeletionStage::Fenced, generation).await?; validate_storage_manifest(manifest)?; // The chunk rows are the concrete delete list; the freeze commits only @@ -1405,22 +1417,29 @@ impl DeletionStore { .map_err(|_| DbError::DeletionSafety("sweep unknown count overflow".to_string()))?; let cap = i64::try_from(object_cap) .map_err(|_| DbError::DeletionSafety("sweep object cap overflow".to_string()))?; - // completed_at is clamped server-side: the sweeper's clock supplies - // started_at, and skew ahead of Postgres would otherwise violate the - // completed_at >= started_at check. - let (id, completed_at): (Uuid, DateTime) = sqlx::query_as( + // Completion is authoritative database time. Small positive sweeper + // skew is clamped at that boundary; materially future starts are rejected. + let row: Option<(Uuid, DateTime, DateTime)> = sqlx::query_as( "INSERT INTO storage_taxonomy_sweeps \ (started_at, completed_at, listed_objects, unknown_object_count, \ unknown_key_sample, object_cap) \ - VALUES ($1, GREATEST(now(), $1), $2, $3, $4, $5) RETURNING id, completed_at", + SELECT LEAST($1, db_now), db_now, $2, $3, $4, $5 \ + FROM (SELECT clock_timestamp() AS db_now) clock \ + WHERE $1 <= db_now + interval '5 minutes' \ + RETURNING id, started_at, completed_at", ) .bind(started_at) .bind(listed) .bind(unknown) .bind(sqlx::types::Json(unknown_key_sample)) .bind(cap) - .fetch_one(&self.pool) + .fetch_optional(&self.pool) .await?; + let (id, started_at, completed_at) = row.ok_or_else(|| { + DbError::DeletionSafety( + "taxonomy sweep start time is more than five minutes in the future".to_string(), + ) + })?; Ok(TaxonomySweep { id, started_at, @@ -3351,6 +3370,30 @@ mod postgres_tests { .contains("tombstones are permanent")); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn taxonomy_sweep_uses_database_completion_order() { + let (_, store) = store().await; + store + .record_taxonomy_sweep(Utc::now() + chrono::Duration::minutes(1), 1, 0, &[], 100) + .await + .expect("record skewed clean sweep"); + let dirty = store + .record_taxonomy_sweep(Utc::now(), 1, 1, &["unknown".to_string()], 100) + .await + .expect("record later dirty sweep"); + + assert_eq!( + store + .latest_taxonomy_sweep() + .await + .expect("latest sweep") + .unwrap() + .id, + dirty.id + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn manifest_key_chunks_bind_freeze_execution_and_cleanup() { @@ -3424,6 +3467,19 @@ mod postgres_tests { .is_err() ); assert!(store.clear_manifest_key_chunks(&token).await.is_err()); + assert!( + sqlx::query( + "INSERT INTO community_deletion_manifest_keys \ + (request_id, chunk_no, prefix, keys) VALUES ($1, 2, $2, $3)", + ) + .bind(request.id) + .bind(&meta_prefix) + .bind(sqlx::types::Json(&keys[..1])) + .execute(&db.pool) + .await + .is_err(), + "the database must reject chunks appended after freeze" + ); store.mark_drained(&token).await.expect("drained"); let first = store diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index e67ef7037a..45505e7288 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -519,8 +519,23 @@ async fn require_fresh_clean_sweep(services: &Services) -> Result .ok_or_else(|| { transient("no storage taxonomy sweep recorded; run the deletions sweep command") })?; - let age = chrono::Utc::now().signed_duration_since(sweep.completed_at); - let max_age = chrono::Duration::from_std(sweep_max_age()).unwrap_or(chrono::Duration::MAX); + validate_fresh_clean_sweep(&sweep, chrono::Utc::now(), sweep_max_age())?; + Ok(sweep) +} + +fn validate_fresh_clean_sweep( + sweep: &TaxonomySweep, + now: chrono::DateTime, + max_age: Duration, +) -> Result<()> { + let age = now.signed_duration_since(sweep.completed_at); + let max_age = chrono::Duration::from_std(max_age).unwrap_or(chrono::Duration::MAX); + if age < chrono::Duration::zero() { + return Err(transient(format!( + "latest storage taxonomy sweep {} completed in the future; run the deletions sweep command", + sweep.id + ))); + } if age > max_age { return Err(transient(format!( "latest storage taxonomy sweep {} is stale ({}h old); run the deletions sweep command", @@ -536,7 +551,7 @@ async fn require_fresh_clean_sweep(services: &Services) -> Result sweep.unknown_key_sample.join(",") ))); } - Ok(sweep) + Ok(()) } fn validate_frozen_inventory(request: &DeletionRequest) -> Result { @@ -1385,6 +1400,22 @@ fn print_json(value: &impl Serialize) -> Result<()> { mod tests { use super::*; + #[test] + fn future_taxonomy_sweep_fails_closed() { + let now = chrono::Utc::now(); + let sweep = TaxonomySweep { + id: Uuid::new_v4(), + started_at: now, + completed_at: now + chrono::Duration::minutes(1), + listed_objects: 0, + unknown_object_count: 0, + unknown_key_sample: Vec::new(), + object_cap: 1_000_000, + }; + + assert!(validate_fresh_clean_sweep(&sweep, now, Duration::from_secs(86_400)).is_err()); + } + fn empty_storage_manifest( community: buzz_core::CommunityId, sweep_id: Uuid, diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 79ffd930eb..f41534b946 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -162,9 +162,11 @@ CREATE TABLE community_deletion_manifest_keys ( ); -- Chunk content is immutable once written; the only permitted update is the --- one-way deleted_at stamp. Removal is permitted only while the destructive --- manifest has not yet frozen (a retried partial freeze rewrites its chunks) --- or once the request has passed logical verification (terminal cleanup). +-- one-way deleted_at stamp. New chunks are permitted only while the request is +-- fenced and its destructive manifest remains unfrozen. Removal is permitted +-- only while the destructive manifest has not yet frozen (a retried partial +-- freeze rewrites its chunks) or once the request has passed logical +-- verification (terminal cleanup). CREATE FUNCTION protect_community_deletion_manifest_keys() RETURNS trigger LANGUAGE plpgsql @@ -188,7 +190,15 @@ BEGIN SELECT destructive_storage_frozen_at, stage INTO frozen_at, request_stage FROM community_deletion_requests - WHERE id = OLD.request_id; + WHERE id = CASE WHEN TG_OP = 'INSERT' THEN NEW.request_id ELSE OLD.request_id END + FOR UPDATE; + IF TG_OP = 'INSERT' THEN + IF FOUND AND frozen_at IS NULL AND request_stage = 'fenced' THEN + RETURN NEW; + END IF; + RAISE EXCEPTION 'community deletion manifest key chunks require an unfrozen fenced request' + USING ERRCODE = 'integrity_constraint_violation'; + END IF; IF NOT FOUND OR frozen_at IS NULL OR request_stage IN ('logically_verified', 'retention_pending') @@ -201,7 +211,7 @@ END; $$; CREATE TRIGGER community_deletion_manifest_keys_guard -BEFORE UPDATE OR DELETE ON community_deletion_manifest_keys +BEFORE INSERT OR UPDATE OR DELETE ON community_deletion_manifest_keys FOR EACH ROW EXECUTE FUNCTION protect_community_deletion_manifest_keys(); From ace0f220c1b724e4bb2dc161996abd1e1c201267 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Mon, 3 Aug 2026 15:20:34 -0700 Subject: [PATCH 24/51] fix: scope community deletion storage inventory Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 62 ++----- crates/buzz-deletion/src/lib.rs | 179 ++++----------------- crates/buzz-media/src/storage.rs | 22 +++ crates/buzz-relay/src/api/git/manifest.rs | 11 ++ crates/buzz-relay/src/api/git/transport.rs | 8 +- crates/buzz-relay/src/handlers/ingest.rs | 8 +- migrations/0028_community_deletion.sql | 7 +- 7 files changed, 79 insertions(+), 218 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 31da75fac2..bb15087409 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -228,7 +228,7 @@ pub struct DeletionRequest { pub reason: Option, /// Frozen catalog manifest. pub schema_manifest: Option, - /// Frozen storage taxonomy manifest observed at submission. + /// Frozen community-prefix storage manifest observed at submission. pub storage_manifest: Option, /// Destructive storage manifest frozen after the durable fence. pub destructive_storage_manifest: Option, @@ -276,20 +276,15 @@ pub struct SchemaManifest { } /// Frozen storage inventory supplied by the object-store adapter: slim -/// per-prefix summaries bound to a clean fleet taxonomy sweep. The concrete -/// key list never lives on the request row — the destructive freeze persists -/// it as chunked `community_deletion_manifest_keys` rows that must hash to -/// these digests. +/// per-prefix summaries for the target community. The concrete key list never +/// lives on the request row — the destructive freeze persists it as chunked +/// `community_deletion_manifest_keys` rows that must hash to these digests. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct StorageManifest { /// Adapter schema version. pub version: i32, /// Per-prefix frozen summaries, strictly sorted by prefix. pub prefixes: Vec, - /// Clean fleet taxonomy sweep this inventory relied on. - pub taxonomy_sweep_id: Uuid, - /// Tenant-prefix keys outside the exact writer taxonomy. Non-empty fails. - pub unknown_keys: Vec, } /// Frozen summary of one community-scoped key prefix. @@ -319,7 +314,7 @@ pub struct ManifestKeyChunk { /// One durable fleet-wide object-store taxonomy sweep record. #[derive(Debug, Clone, Serialize)] pub struct TaxonomySweep { - /// Sweep identity bound into storage manifests. + /// Sweep identity. pub id: Uuid, /// Listing start time. pub started_at: DateTime, @@ -1250,8 +1245,8 @@ impl DeletionStore { validate_storage_manifest(manifest)?; // The chunk rows are the concrete delete list; the freeze commits only // if they hash to the manifest's frozen per-prefix digests. Loading the - // full chunk stream is a one-time freeze-boundary cost bounded by the - // per-community object cap, not the fleet. + // full chunk stream is a one-time freeze-boundary cost proportional to + // this community's bindings, never the fleet bucket. let chunks: Vec<(i64, String, sqlx::types::Json>)> = sqlx::query_as( "SELECT chunk_no, prefix, keys FROM community_deletion_manifest_keys \ WHERE request_id = $1 ORDER BY chunk_no", @@ -2195,9 +2190,9 @@ fn validate_serving_migration_version(migration_version: Option) -> Result< } } -/// Fail closed when storage inventory reports unknown or malformed data. +/// Fail closed when a community-prefix inventory has an unsafe shape. pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { - if manifest.version != 3 { + if manifest.version != 4 { return Err(DbError::DeletionSafety(format!( "unsupported storage manifest version {}", manifest.version @@ -2236,17 +2231,6 @@ pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { ))); } } - if manifest.taxonomy_sweep_id.is_nil() { - return Err(DbError::DeletionSafety( - "storage manifest is not bound to a taxonomy sweep".to_string(), - )); - } - if !manifest.unknown_keys.is_empty() { - return Err(DbError::DeletionSafety(format!( - "unknown object-store keys block deletion: {}", - manifest.unknown_keys.join(",") - ))); - } Ok(()) } @@ -2571,14 +2555,12 @@ mod tests { fn storage_manifest() -> StorageManifest { StorageManifest { - version: 3, + version: 4, prefixes: vec![ empty_prefix("_meta/c/"), empty_prefix("_uploads/c/"), empty_prefix("repos/c/"), ], - taxonomy_sweep_id: Uuid::from_u128(1), - unknown_keys: Vec::new(), } } @@ -2654,18 +2636,6 @@ mod tests { let mut malformed_digest = storage_manifest(); malformed_digest.prefixes[0].keys_digest = "not-hex".to_string(); assert!(validate_storage_manifest(&malformed_digest).is_err()); - - let mut unswept = storage_manifest(); - unswept.taxonomy_sweep_id = Uuid::nil(); - assert!(validate_storage_manifest(&unswept).is_err()); - } - - #[test] - fn unknown_storage_data_fails_closed() { - let mut manifest = storage_manifest(); - manifest.unknown_keys.push("mystery/data".to_string()); - let error = validate_storage_manifest(&manifest).expect_err("unknown key must block"); - assert!(error.to_string().contains("unknown object-store keys")); } #[test] @@ -2785,16 +2755,14 @@ mod postgres_tests { } } - fn empty_storage_manifest(community: CommunityId, sweep_id: Uuid) -> StorageManifest { + fn empty_storage_manifest(community: CommunityId) -> StorageManifest { StorageManifest { - version: 3, + version: 4, prefixes: vec![ empty_prefix_manifest(format!("_meta/{community}/")), empty_prefix_manifest(format!("_uploads/{community}/")), empty_prefix_manifest(format!("repos/{community}/")), ], - taxonomy_sweep_id: sweep_id, - unknown_keys: Vec::new(), } } @@ -2812,16 +2780,12 @@ mod postgres_tests { .await .expect("submit"); assert_eq!(submitted.community_id, community.id); - let sweep = store - .record_taxonomy_sweep(Utc::now(), 0, 0, &[], 1_000_000) - .await - .expect("record clean taxonomy sweep"); let inventory = FrozenInventory { schema: store .inventory_schema(community.id) .await .expect("schema inventory"), - storage: empty_storage_manifest(community.id, sweep.id), + storage: empty_storage_manifest(community.id), }; let request = store .freeze_inventory(submitted.id, &inventory) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 45505e7288..7eaa2bfb40 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -10,8 +10,7 @@ use std::time::Duration; use anyhow::{Context, Result}; use buzz_db::deletion::{ ClaimedDeletion, DeletionRequest, DeletionStage, DeletionStore, FrozenInventory, - KeyStreamDigest, LeaseToken, PrefixManifest, StorageManifest, TaxonomySweep, - DEFAULT_LEASE_DURATION, + KeyStreamDigest, LeaseToken, PrefixManifest, StorageManifest, DEFAULT_LEASE_DURATION, }; use buzz_db::{Db, DbConfig}; use buzz_media::{is_tenant_owned_key, tenant_prefixes, MediaStorage}; @@ -20,12 +19,8 @@ use serde::Serialize; use tokio_util::sync::CancellationToken; use uuid::Uuid; -/// Per-community object cap; the fleet cap moved to the taxonomy sweep. -const DEFAULT_COMMUNITY_OBJECT_CAP: u64 = 1_000_000; -/// Fleet-wide object cap for one taxonomy sweep. +/// Fleet-wide object cap for one observational taxonomy sweep. const DEFAULT_SWEEP_OBJECT_CAP: u64 = 10_000_000; -/// A deletion may rely on a clean sweep at most this old. -const DEFAULT_SWEEP_MAX_AGE: Duration = Duration::from_secs(24 * 60 * 60); /// Keys per frozen side-table chunk (one `DeleteObjects`-sized unit × 10). const DEFAULT_MANIFEST_CHUNK_KEYS: usize = 10_000; /// Unknown keys retained verbatim on a sweep record for diagnosis. @@ -271,11 +266,10 @@ pub enum Command { #[arg(long)] executor_id: Option, }, - /// Sweep the whole bucket's key taxonomy and record the fleet evidence. + /// Sweep the whole bucket's key taxonomy and record observational evidence. /// - /// Submit and fencing require a recent clean sweep instead of re-listing - /// the fleet bucket per deletion. Exits non-zero when unknown writer - /// shapes are found. + /// This is independent of community deletion. It reports unknown writer + /// shapes but never gates submission or destructive progress. Sweep, } @@ -363,12 +357,11 @@ pub async fn run(command: Command) -> Result { if host.is_empty() { anyhow::bail!("cannot derive community host; pass --host or set RELAY_URL"); } - let sweep = require_fresh_clean_sweep(&services).await?; let request = services .store .submit(&host, &requested_by, reason.as_deref()) .await?; - let inventory = build_inventory(&services, &request, sweep.id).await?; + let inventory = build_inventory(&services, &request).await?; let request = services .store .freeze_inventory(request.id, &inventory) @@ -506,54 +499,6 @@ where .unwrap_or(default) } -/// Gate destructive progress on recent fleet-wide taxonomy evidence. -/// -/// A missing or stale sweep is transient — the operator runs the sweep -/// command and retries. A dirty sweep is the fail-closed unknown-key -/// invariant and blocks permanently. -async fn require_fresh_clean_sweep(services: &Services) -> Result { - let sweep = services - .store - .latest_taxonomy_sweep() - .await? - .ok_or_else(|| { - transient("no storage taxonomy sweep recorded; run the deletions sweep command") - })?; - validate_fresh_clean_sweep(&sweep, chrono::Utc::now(), sweep_max_age())?; - Ok(sweep) -} - -fn validate_fresh_clean_sweep( - sweep: &TaxonomySweep, - now: chrono::DateTime, - max_age: Duration, -) -> Result<()> { - let age = now.signed_duration_since(sweep.completed_at); - let max_age = chrono::Duration::from_std(max_age).unwrap_or(chrono::Duration::MAX); - if age < chrono::Duration::zero() { - return Err(transient(format!( - "latest storage taxonomy sweep {} completed in the future; run the deletions sweep command", - sweep.id - ))); - } - if age > max_age { - return Err(transient(format!( - "latest storage taxonomy sweep {} is stale ({}h old); run the deletions sweep command", - sweep.id, - age.num_hours() - ))); - } - if sweep.unknown_object_count > 0 { - return Err(permanent(format!( - "latest storage taxonomy sweep {} found {} unknown object-store key(s); sample: {}", - sweep.id, - sweep.unknown_object_count, - sweep.unknown_key_sample.join(",") - ))); - } - Ok(()) -} - fn validate_frozen_inventory(request: &DeletionRequest) -> Result { let frozen: FrozenInventory = serde_json::from_value( request @@ -593,13 +538,12 @@ fn validate_storage_ownership(request: &DeletionRequest, manifest: &StorageManif async fn build_inventory( services: &Services, request: &DeletionRequest, - sweep_id: Uuid, ) -> Result { let schema = services .store .inventory_schema(request.community_id) .await?; - let storage = enumerate_tenant_prefixes(services, request, sweep_id, None, None).await?; + let storage = enumerate_tenant_prefixes(services, request, None, None).await?; Ok(FrozenInventory { schema, storage }) } @@ -625,16 +569,16 @@ async fn flush_chunk(services: &Services, sink: &mut ChunkSink<'_>, prefix: &str /// Enumerate the target's three tenant prefixes into per-prefix summaries. /// -/// Cost is O(tenant objects) regardless of fleet size; the fleet-level -/// unknown-key invariant is covered by the taxonomy sweep gate instead. -/// Memory stays bounded at one listing page plus one buffered chunk — the -/// full key list is never materialized. When `sink` is supplied, keys are -/// also persisted as side-table chunks (never spanning prefixes) for the -/// destructive freeze to bind against these digests. +/// Cost is O(tenant objects) regardless of fleet size. Unknown shapes inside +/// one of the owned prefixes fail closed; keys elsewhere in the shared bucket +/// are outside this operation's contract. Memory stays bounded at one listing +/// page plus one buffered chunk — the full key list is never materialized. +/// When `sink` is supplied, keys are also persisted as side-table chunks +/// (never spanning prefixes) for the destructive freeze to bind against these +/// digests. async fn enumerate_tenant_prefixes( services: &Services, request: &DeletionRequest, - sweep_id: Uuid, heartbeat_lost: Option<&CancellationToken>, mut sink: Option<&mut ChunkSink<'_>>, ) -> Result { @@ -644,10 +588,8 @@ async fn enumerate_tenant_prefixes( )); } let community = *request.community_id.as_uuid(); - let cap = storage_object_cap(); let chunk_keys = manifest_chunk_keys(); let mut prefixes = Vec::new(); - let mut total_objects: u64 = 0; for prefix in tenant_prefixes(community) { let mut digest = KeyStreamDigest::new(); let mut total_bytes: u64 = 0; @@ -660,12 +602,6 @@ async fn enumerate_tenant_prefixes( .media .list_prefix_page(&prefix, continuation.take(), LIST_PAGE_SIZE) .await?; - total_objects = total_objects.saturating_add(page.objects.len() as u64); - if total_objects > cap { - return Err(permanent(format!( - "community storage inventory exceeds per-community object cap {cap}" - ))); - } for (key, size) in page.objects { if !is_tenant_owned_key(community, &key) { return Err(permanent(format!( @@ -703,10 +639,8 @@ async fn enumerate_tenant_prefixes( }); } let manifest = StorageManifest { - version: 3, + version: 4, prefixes, - taxonomy_sweep_id: sweep_id, - unknown_keys: Vec::new(), }; buzz_db::deletion::validate_storage_manifest(&manifest)?; Ok(manifest) @@ -721,7 +655,7 @@ async fn freeze_destructive_manifest( token: &LeaseToken, heartbeat_lost: &CancellationToken, ) -> Result { - let frozen = validate_frozen_inventory(request)?; + validate_frozen_inventory(request)?; // A prior interrupted freeze may have left partial chunks; they were // never bound to a committed manifest, so rewrite them from scratch. services.store.clear_manifest_key_chunks(token).await?; @@ -730,14 +664,8 @@ async fn freeze_destructive_manifest( next_chunk_no: 0, buffered: Vec::new(), }; - let manifest = enumerate_tenant_prefixes( - services, - request, - frozen.storage.taxonomy_sweep_id, - Some(heartbeat_lost), - Some(&mut sink), - ) - .await?; + let manifest = + enumerate_tenant_prefixes(services, request, Some(heartbeat_lost), Some(&mut sink)).await?; services .store .freeze_destructive_storage_manifest(token, &manifest) @@ -1009,9 +937,9 @@ async fn execute_stage( } match request.stage { DeletionStage::Approved => { - // Approval binds immutable catalog + key taxonomy. Live row counts - // and tenant binding keys are deliberately not equality-bound until - // the durable fence closes all writers. + // Approval binds immutable catalog + community-prefix ownership. + // Live row counts and tenant binding keys are deliberately not + // equality-bound until the durable fence closes all writers. let live_schema = services .store .inventory_schema(request.community_id) @@ -1022,9 +950,6 @@ async fn execute_stage( "approved structural catalog drifted before fencing", )); } - // The fleet unknown-key invariant is sweep evidence, not an - // inline re-listing of the whole bucket. - require_fresh_clean_sweep(services).await?; services.store.begin_quiescing(&token).await?; match services.store.fence(&token).await { Ok(_) => {} @@ -1319,14 +1244,6 @@ async fn verify_redis_absence( } } -fn storage_object_cap() -> u64 { - std::env::var("BUZZ_DELETION_STORAGE_MAX_OBJECTS") - .ok() - .and_then(|value| value.parse().ok()) - .filter(|value| *value > 0) - .unwrap_or(DEFAULT_COMMUNITY_OBJECT_CAP) -} - fn sweep_object_cap() -> u64 { std::env::var("BUZZ_DELETION_SWEEP_MAX_OBJECTS") .ok() @@ -1335,15 +1252,6 @@ fn sweep_object_cap() -> u64 { .unwrap_or(DEFAULT_SWEEP_OBJECT_CAP) } -fn sweep_max_age() -> Duration { - std::env::var("BUZZ_DELETION_SWEEP_MAX_AGE_SECS") - .ok() - .and_then(|value| value.parse().ok()) - .filter(|value| *value > 0) - .map(Duration::from_secs) - .unwrap_or(DEFAULT_SWEEP_MAX_AGE) -} - fn manifest_chunk_keys() -> usize { std::env::var("BUZZ_DELETION_MANIFEST_CHUNK_KEYS") .ok() @@ -1400,28 +1308,9 @@ fn print_json(value: &impl Serialize) -> Result<()> { mod tests { use super::*; - #[test] - fn future_taxonomy_sweep_fails_closed() { - let now = chrono::Utc::now(); - let sweep = TaxonomySweep { - id: Uuid::new_v4(), - started_at: now, - completed_at: now + chrono::Duration::minutes(1), - listed_objects: 0, - unknown_object_count: 0, - unknown_key_sample: Vec::new(), - object_cap: 1_000_000, - }; - - assert!(validate_fresh_clean_sweep(&sweep, now, Duration::from_secs(86_400)).is_err()); - } - - fn empty_storage_manifest( - community: buzz_core::CommunityId, - sweep_id: Uuid, - ) -> StorageManifest { + fn empty_storage_manifest(community: buzz_core::CommunityId) -> StorageManifest { StorageManifest { - version: 3, + version: 4, prefixes: tenant_prefixes(*community.as_uuid()) .into_iter() .map(|prefix| PrefixManifest { @@ -1431,8 +1320,6 @@ mod tests { keys_digest: KeyStreamDigest::new().finish().0, }) .collect(), - taxonomy_sweep_id: sweep_id, - unknown_keys: Vec::new(), } } @@ -1455,16 +1342,12 @@ mod tests { .submit(&host, "test", None) .await .expect("submit deletion request"); - let sweep = store - .record_taxonomy_sweep(chrono::Utc::now(), 0, 0, &[], 1_000_000) - .await - .expect("record clean taxonomy sweep"); let inventory = FrozenInventory { schema: store .inventory_schema(community.id) .await .expect("inventory schema"), - storage: empty_storage_manifest(community.id, sweep.id), + storage: empty_storage_manifest(community.id), }; store .freeze_inventory(request.id, &inventory) @@ -1552,16 +1435,14 @@ mod tests { let mut digest_tampered = claim.request.clone(); digest_tampered.inventory_manifest = Some(serde_json::json!({ "schema": {"revision": 0, "migration_version": 0, "scoped_tables": [], "row_counts": {}, "fenced_tables": []}, - "storage": {"version": 3, "prefixes": [], "taxonomy_sweep_id": Uuid::from_u128(1), "unknown_keys": []} + "storage": {"version": 4, "prefixes": []} })); assert!(validate_frozen_inventory(&digest_tampered).is_err()); // A manifest scoped to another community's prefixes is never the // deletion target's, even when internally valid. - let foreign_manifest = empty_storage_manifest( - buzz_core::CommunityId::from_uuid(Uuid::new_v4()), - Uuid::from_u128(1), - ); + let foreign_manifest = + empty_storage_manifest(buzz_core::CommunityId::from_uuid(Uuid::new_v4())); assert!(validate_storage_ownership(&claim.request, &foreign_manifest).is_err()); } @@ -1614,11 +1495,7 @@ mod tests { digest.fold(key).expect("fold key"); } let (keys_digest, object_count) = digest.finish(); - let mut storage = empty_storage_manifest(community, Uuid::from_u128(1)); - storage.taxonomy_sweep_id = validate_frozen_inventory(&claim.request) - .expect("frozen inventory") - .storage - .taxonomy_sweep_id; + let mut storage = empty_storage_manifest(community); storage.prefixes[0] = PrefixManifest { prefix: meta_prefix.clone(), object_count, diff --git a/crates/buzz-media/src/storage.rs b/crates/buzz-media/src/storage.rs index f9b642b5ae..0f0aa7af62 100644 --- a/crates/buzz-media/src/storage.rs +++ b/crates/buzz-media/src/storage.rs @@ -496,6 +496,28 @@ mod tests { ); } + #[test] + fn tenant_key_writers_are_covered_by_deletion_taxonomy() { + let ctx = tenant(1); + let community = *ctx.community().as_uuid(); + let sha = "a".repeat(64); + let event_id = "01ARZ3NDEKTSV4RRFFQ69G5FAV"; + let sidecar = MediaStorage::ctx_sidecar_key(&ctx, &sha); + let upload = crate::upload_record::upload_record_key(&ctx, &sha, event_id); + let prefixes = crate::bucket_index::tenant_prefixes(community); + + for key in [sidecar, upload] { + assert!( + prefixes.iter().any(|prefix| key.starts_with(prefix)), + "tenant writer key {key} is outside deletion prefixes" + ); + assert!( + crate::bucket_index::is_tenant_owned_key(community, &key), + "tenant writer key {key} is not recognized by deletion taxonomy" + ); + } + } + #[test] fn sidecar_keys_are_community_scoped() { let a = tenant(1); diff --git a/crates/buzz-relay/src/api/git/manifest.rs b/crates/buzz-relay/src/api/git/manifest.rs index baf109c1ad..0dfbdb35a4 100644 --- a/crates/buzz-relay/src/api/git/manifest.rs +++ b/crates/buzz-relay/src/api/git/manifest.rs @@ -474,6 +474,17 @@ mod tests { m.validate().expect("no parent is fine (first push)"); } + #[test] + fn pointer_writer_is_covered_by_deletion_taxonomy() { + let community = CommunityId::from_uuid(uuid::Uuid::from_u128(1)); + let owner = "a".repeat(64); + let key = pointer_key(community, &owner, "repo"); + let prefixes = buzz_media::tenant_prefixes(*community.as_uuid()); + + assert!(prefixes.iter().any(|prefix| key.starts_with(prefix))); + assert!(buzz_media::is_tenant_owned_key(*community.as_uuid(), &key)); + } + #[test] fn pointer_key_strips_dot_git() { let c = CommunityId::from_uuid(uuid::Uuid::from_u128(1)); diff --git a/crates/buzz-relay/src/api/git/transport.rs b/crates/buzz-relay/src/api/git/transport.rs index 556abcad84..3b2241046a 100644 --- a/crates/buzz-relay/src/api/git/transport.rs +++ b/crates/buzz-relay/src/api/git/transport.rs @@ -2336,17 +2336,13 @@ mod track_c_tests { .submit(host, "git-finalize-test", Some("post-CAS lease regression")) .await .expect("submit deletion"); - let sweep = store - .record_taxonomy_sweep(chrono::Utc::now(), 0, 0, &[], 1_000_000) - .await - .expect("record clean taxonomy sweep"); let inventory = FrozenInventory { schema: store .inventory_schema(request.community_id) .await .expect("schema inventory"), storage: StorageManifest { - version: 3, + version: 4, prefixes: buzz_media::tenant_prefixes(*request.community_id.as_uuid()) .into_iter() .map(|prefix| PrefixManifest { @@ -2356,8 +2352,6 @@ mod track_c_tests { keys_digest: KeyStreamDigest::new().finish().0, }) .collect(), - taxonomy_sweep_id: sweep.id, - unknown_keys: Vec::new(), }, }; store diff --git a/crates/buzz-relay/src/handlers/ingest.rs b/crates/buzz-relay/src/handlers/ingest.rs index 947128b307..ed721872b5 100644 --- a/crates/buzz-relay/src/handlers/ingest.rs +++ b/crates/buzz-relay/src/handlers/ingest.rs @@ -3197,17 +3197,13 @@ mod tests { ) .await .expect("submit"); - let sweep = store - .record_taxonomy_sweep(chrono::Utc::now(), 0, 0, &[], 1_000_000) - .await - .expect("record clean taxonomy sweep"); let inventory = FrozenInventory { schema: store .inventory_schema(community) .await .expect("schema inventory"), storage: StorageManifest { - version: 3, + version: 4, prefixes: buzz_media::tenant_prefixes(*community.as_uuid()) .into_iter() .map(|prefix| PrefixManifest { @@ -3217,8 +3213,6 @@ mod tests { keys_digest: KeyStreamDigest::new().finish().0, }) .collect(), - taxonomy_sweep_id: sweep.id, - unknown_keys: Vec::new(), }, }; let request = store diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index f41534b946..21b71f378b 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -215,10 +215,9 @@ BEFORE INSERT OR UPDATE OR DELETE ON community_deletion_manifest_keys FOR EACH ROW EXECUTE FUNCTION protect_community_deletion_manifest_keys(); --- Fleet-wide object-store taxonomy sweep evidence. The unknown-key --- fail-closed invariant is a fleet property, not a per-request one: deletion --- inventory and fencing require a recent clean sweep instead of re-listing --- the whole bucket inline per request. +-- Fleet-wide object-store taxonomy sweep evidence. This is an independent +-- observability record: community deletion inventories only the target's owned +-- prefixes and does not gate submission or execution on sweep state. CREATE TABLE storage_taxonomy_sweeps ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), started_at TIMESTAMPTZ NOT NULL, From 86b71fc932c8a718ccef2237553221bbcbe7864b Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Tue, 4 Aug 2026 09:36:28 -0700 Subject: [PATCH 25/51] fix: decouple deletion operator commands Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-deletion/src/lib.rs | 125 +++++++++++++++++++++++--------- 1 file changed, 92 insertions(+), 33 deletions(-) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 7eaa2bfb40..c275e2baf1 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -341,22 +341,39 @@ struct RunOutput { /// Execute one nested deletion command. pub async fn run(command: Command) -> Result { - let services = connect_services().await?; + match command { + Command::List { limit } => { + let store = connect_store().await?; + print_json(&store.list(i64::from(limit)).await?)?; + Ok(0) + } + Command::Inspect { id } => { + let store = connect_store().await?; + print_json(&store.inspect(id).await?)?; + Ok(0) + } + Command::Approve { + id, + approved_by, + note, + } => { + let store = connect_store().await?; + print_json(&store.approve(id, &approved_by, note.as_deref()).await?)?; + Ok(0) + } + command => run_with_services(command, connect_services().await?).await, + } +} + +async fn run_with_services(command: Command, services: Services) -> Result { match command { Command::Submit { host, requested_by, reason, } => { - let host = host.unwrap_or_else(|| { - buzz_core::tenant::relay_url_authority( - &std::env::var("RELAY_URL") - .unwrap_or_else(|_| "ws://localhost:3000".to_string()), - ) - }); - if host.is_empty() { - anyhow::bail!("cannot derive community host; pass --host or set RELAY_URL"); - } + let relay_url = std::env::var("RELAY_URL").ok(); + let host = resolve_submit_host(host.as_deref(), relay_url.as_deref())?; let request = services .store .submit(&host, &requested_by, reason.as_deref()) @@ -369,27 +386,6 @@ pub async fn run(command: Command) -> Result { print_json(&request)?; Ok(0) } - Command::List { limit } => { - print_json(&services.store.list(i64::from(limit)).await?)?; - Ok(0) - } - Command::Inspect { id } => { - print_json(&services.store.inspect(id).await?)?; - Ok(0) - } - Command::Approve { - id, - approved_by, - note, - } => { - print_json( - &services - .store - .approve(id, &approved_by, note.as_deref()) - .await?, - )?; - Ok(0) - } Command::Run { id, executor_id } => { run_loop( services, @@ -431,10 +427,37 @@ pub async fn run(command: Command) -> Result { print_json(&sweep)?; Ok(i32::from(sweep.unknown_object_count > 0)) } + Command::List { .. } | Command::Inspect { .. } | Command::Approve { .. } => { + anyhow::bail!("database-only command reached full-service dispatcher") + } } } -async fn connect_services() -> Result { +fn resolve_submit_host(host: Option<&str>, relay_url: Option<&str>) -> Result { + if let Some(host) = host { + let host = host.trim(); + if host.is_empty() { + anyhow::bail!("--host must not be empty"); + } + return Ok(host.to_owned()); + } + + let relay_url = relay_url + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + anyhow::anyhow!("cannot derive community host; pass --host or set RELAY_URL") + })?; + let host = buzz_core::tenant::relay_url_authority(relay_url); + if host.is_empty() { + anyhow::bail!( + "cannot derive community host from RELAY_URL; pass --host or set a valid RELAY_URL" + ); + } + Ok(host) +} + +async fn connect_store() -> Result { let database_url = required_env("DATABASE_URL")?; let db = Db::new(&DbConfig { database_url, @@ -442,7 +465,11 @@ async fn connect_services() -> Result { ..DbConfig::default() }) .await?; - let store = store(&db); + Ok(store(&db)) +} + +async fn connect_services() -> Result { + let store = connect_store().await?; let media_config = buzz_media::MediaConfig { s3_endpoint: required_env("BUZZ_S3_ENDPOINT")?, s3_access_key: required_env("BUZZ_S3_ACCESS_KEY")?, @@ -1308,6 +1335,38 @@ fn print_json(value: &impl Serialize) -> Result<()> { mod tests { use super::*; + #[test] + fn submit_host_prefers_explicit_host() { + assert_eq!( + resolve_submit_host(Some(" community.example "), Some("wss://ignored.example")) + .expect("explicit host"), + "community.example" + ); + } + + #[test] + fn submit_host_derives_from_relay_url() { + assert_eq!( + resolve_submit_host(None, Some("wss://relay.example:8443/path")) + .expect("relay URL host"), + "relay.example:8443" + ); + } + + #[test] + fn submit_host_requires_an_explicit_source() { + for relay_url in [None, Some(""), Some(" ")] { + let error = resolve_submit_host(None, relay_url).expect_err("missing host must fail"); + assert!(error.to_string().contains("pass --host or set RELAY_URL")); + } + } + + #[test] + fn submit_host_rejects_empty_or_invalid_values() { + assert!(resolve_submit_host(Some(" "), Some("wss://relay.example")).is_err()); + assert!(resolve_submit_host(None, Some("not a URL")).is_err()); + } + fn empty_storage_manifest(community: buzz_core::CommunityId) -> StorageManifest { StorageManifest { version: 4, From 1c311276de02977b1f356108bec6216ad8814c17 Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Tue, 4 Aug 2026 12:29:40 -0700 Subject: [PATCH 26/51] test: restore topology-safe live relay fixtures Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-test-client/tests/e2e_relay.rs | 114 ++++++++++++--------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/crates/buzz-test-client/tests/e2e_relay.rs b/crates/buzz-test-client/tests/e2e_relay.rs index 5b9a50b5b1..a8adcc15ef 100644 --- a/crates/buzz-test-client/tests/e2e_relay.rs +++ b/crates/buzz-test-client/tests/e2e_relay.rs @@ -120,7 +120,12 @@ async fn seed_relay_member(host: &str, keys: &Keys, role: &str) { } async fn seed_relay_owner(keys: &Keys) { - seed_relay_member("localhost:3000", keys, "owner").await; + seed_relay_member(&relay_authority(), keys, "owner").await; +} + +fn relay_authority() -> String { + let url = url::Url::parse(&relay_http_url()).expect("relay HTTP URL"); + url[url::Position::BeforeHost..url::Position::AfterPort].to_string() } fn http_origin_for_host(host: &str) -> String { @@ -315,7 +320,7 @@ async fn test_invite_claim_rejects_invalid_code() { #[ignore] async fn test_invite_mint_requires_owner_or_admin() { let member = Keys::generate(); - seed_relay_member("localhost:3000", &member, "member").await; + seed_relay_member(&relay_authority(), &member, "member").await; let response = invite_post(&member, "/api/invites", "{}").await; assert_eq!(response.status(), reqwest::StatusCode::FORBIDDEN); @@ -791,10 +796,10 @@ async fn test_auth_event_kind_rejected() { /// NIP-11 max_subscriptions must be enforced; (limit+1)th REQ gets CLOSED. /// -/// The relay's MAX_SUBSCRIPTIONS is 1024. Opening 1024 subs in a test is slow, -/// so we open a smaller batch and verify the NIP-11 advertised limit matches -/// the actual enforcement constant. The full-limit test is covered by the -/// NIP-11 assertion below (which verifies the advertised value is 1024). +/// This is a protocol-cap test, not an admission-throughput test. Open one REQ +/// at a time and wait out any shared fixed-window quota before retrying a REQ +/// rejected specifically as `rate-limited`, so production admission remains +/// enabled while the test deterministically reaches the independent 1024 cap. #[tokio::test] #[ignore] async fn test_subscription_limit_enforced() { @@ -802,60 +807,75 @@ async fn test_subscription_limit_enforced() { let keys = Keys::generate(); let mut client = BuzzTestClient::connect(&url, &keys).await.expect("connect"); - // Open 1024 subscriptions (the relay's MAX_SUBSCRIPTIONS). for i in 0..1024 { let sid = format!("limit-sub-{i}"); - let filter = Filter::new().kind(Kind::Custom(9)); - client - .subscribe(&sid, vec![filter]) - .await - .expect("subscribe"); - // Drain EOSE to avoid buffer buildup. - client - .collect_until_eose(&sid, Duration::from_secs(5)) - .await - .expect("EOSE"); + let filter = Filter::new().kind(Kind::Custom(49_999)); + subscribe_until_eose(&mut client, &sid, filter).await; } let overflow_sid = sub_id("overflow"); - // Use a kind that no other test writes, so we don't receive stale events. - let filter = Filter::new().kind(Kind::Custom(49999)); - client - .subscribe(&overflow_sid, vec![filter]) - .await - .expect("send REQ"); - - // Drain EOSE and stale events from the 100 earlier subscriptions - // until we receive the CLOSED for the overflow subscription. - let msg = loop { - let m = client - .recv_event(Duration::from_secs(5)) + let filter = Filter::new().kind(Kind::Custom(49_999)); + loop { + client + .subscribe(&overflow_sid, vec![filter.clone()]) .await - .expect("recv CLOSED (or timeout)"); - match &m { - RelayMessage::Eose { .. } => continue, - RelayMessage::Event { .. } => continue, // stale event from earlier subs - _ => break m, - } - }; + .expect("send overflow REQ"); - match msg { - RelayMessage::Closed { - subscription_id, - message, - } => { - assert_eq!(subscription_id, overflow_sid); - assert!( - message.to_lowercase().contains("too many"), - "Expected 'too many' in CLOSED message, got: {message}" - ); + match client + .recv_event(Duration::from_secs(6)) + .await + .expect("recv overflow CLOSED") + { + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id == overflow_sid && message.starts_with("rate-limited:") => { + tokio::time::sleep(Duration::from_secs(5)).await; + } + RelayMessage::Closed { + subscription_id, + message, + } => { + assert_eq!(subscription_id, overflow_sid); + assert!( + message.to_lowercase().contains("too many"), + "Expected 'too many' in CLOSED message, got: {message}" + ); + break; + } + other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } - other => panic!("Expected CLOSED for overflow subscription, got {other:?}"), } client.disconnect().await.expect("disconnect"); } +async fn subscribe_until_eose(client: &mut BuzzTestClient, sid: &str, filter: Filter) { + loop { + client + .subscribe(sid, vec![filter.clone()]) + .await + .expect("subscribe"); + match client + .recv_event(Duration::from_secs(6)) + .await + .expect("EOSE or rate-limit CLOSED") + { + RelayMessage::Eose { subscription_id } => { + assert_eq!(subscription_id, sid); + return; + } + RelayMessage::Closed { + subscription_id, + message, + } if subscription_id == sid && message.starts_with("rate-limited:") => { + tokio::time::sleep(Duration::from_secs(5)).await; + } + other => panic!("unexpected response while opening {sid}: {other:?}"), + } + } +} + #[tokio::test] #[ignore] async fn test_nip11_relay_info() { From ed5a597d87b518c165eed4c34caf40bb7b466b2c Mon Sep 17 00:00:00 2001 From: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Tue, 4 Aug 2026 12:30:58 -0700 Subject: [PATCH 27/51] fix: advance community deletion migration to 0028 Co-authored-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: npub1dccv64krpcpse5cmkzfeh998cftungyatw3djt8jwdw6g43f7fyqzzmrf7 <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/migration.rs | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index ac3b564db9..af1bf8cb46 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -501,7 +501,7 @@ mod tests { migrations.sort_by_key(|migration| migration.version); migrations .into_iter() - .filter(|migration| migration.version > 27) + .filter(|migration| migration.version > 28) .flat_map(|migration| { let statements = split_sql_statements(migration.sql.as_str()); let attachments = statements @@ -668,7 +668,13 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); +<<<<<<< HEAD assert_eq!(migrations.len(), 29); +||||||| parent of 2ea419e57 (fix: advance community deletion migration to 0028) + assert_eq!(migrations.len(), 27); +======= + assert_eq!(migrations.len(), 28); +>>>>>>> 2ea419e57 (fix: advance community deletion migration to 0028) assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1027,6 +1033,7 @@ mod tests { assert!(heartbeat.contains("INSERT INTO replica_heartbeat (id) VALUES (1)")); assert!(heartbeat.contains("_operator_global_tables")); +<<<<<<< HEAD // Channel-id lookup index (0027). let channel_id_index = migrations @@ -1037,6 +1044,24 @@ mod tests { assert!(channel_id_index.sql.as_str().contains("INCLUDE (community_id)")); assert!(channel_id_index.sql.as_str().contains("WHERE deleted_at IS NULL")); assert!(!channel_id_index.sql.as_str().contains("CREATE UNIQUE INDEX")); +||||||| parent of 2ea419e57 (fix: advance community deletion migration to 0028) + + // Channel-id lookup index (0027): serves tenant-independent channel lookups. + assert_eq!(migrations[26].version, 27); + let channel_id_index = migrations[26].sql.as_str(); + assert!(channel_id_index.contains("idx_channels_id_live")); + assert!(channel_id_index.contains("INCLUDE (community_id)")); + assert!(channel_id_index.contains("WHERE deleted_at IS NULL")); + assert!(!channel_id_index.contains("CREATE UNIQUE INDEX")); +======= + // Channel-id lookup index (0027): serves tenant-independent channel lookups. + assert_eq!(migrations[26].version, 27); + let channel_id_index = migrations[26].sql.as_str(); + assert!(channel_id_index.contains("idx_channels_id_live")); + assert!(channel_id_index.contains("INCLUDE (community_id)")); + assert!(channel_id_index.contains("WHERE deleted_at IS NULL")); + assert!(!channel_id_index.contains("CREATE UNIQUE INDEX")); +>>>>>>> 2ea419e57 (fix: advance community deletion migration to 0028) assert!(desired_schema.contains("idx_channels_id_live")); // Main owns 0028 for long reaction payloads. @@ -1359,7 +1384,7 @@ mod tests { run_migrations(&pool) .await .expect("retry succeeds after operator repair"); - assert_eq!(applied_versions(&pool).await.last().copied(), Some(27)); + assert_eq!(applied_versions(&pool).await.last().copied(), Some(28)); } #[tokio::test] From 8d51c847758f57f4bd5d306b6089c14e6b021c8c Mon Sep 17 00:00:00 2001 From: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Wed, 5 Aug 2026 08:54:31 -0700 Subject: [PATCH 28/51] refactor: decouple deletion safety from migration revisions Remove migration-version and catalog-revision coupling while retaining live schema-shape, scoped-table, write-fence, and approval-bound inventory checks. Co-authored-by: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 77 ++------------------------------- crates/buzz-deletion/src/lib.rs | 2 +- 2 files changed, 4 insertions(+), 75 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index bb15087409..4ba2d98517 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -18,10 +18,6 @@ use uuid::Uuid; use crate::error::{DbError, Result}; -/// Exact desired catalog revision understood by this deletion engine. -pub const CATALOG_REVISION: i32 = 1; -/// Highest SQL migration version whose tenant catalog this engine understands. -pub const EXPECTED_MIGRATION_VERSION: i64 = 27; /// Default PostgreSQL lease duration for one claimed deletion request. pub const DEFAULT_LEASE_DURATION: Duration = Duration::from_secs(60); /// Durable name of the schema manifest's PostgreSQL component. @@ -263,10 +259,6 @@ pub struct DeletionRequest { /// Frozen PostgreSQL catalog inventory. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct SchemaManifest { - /// Engine catalog revision. - pub revision: i32, - /// Live SQLx migration version. - pub migration_version: i64, /// Sorted community-scoped table names. pub scoped_tables: Vec, /// Per-table row counts for the target. @@ -657,14 +649,8 @@ impl DeletionStore { }) } - /// Validate the minimum deletion-fence catalog required by relay serving. - /// - /// Serving binaries accept newer additive migrations for rolling upgrades - /// and rollback. Destructive inventory still calls [`Self::validate_catalog`] - /// and requires exact migration/table equality. + /// Validate the deletion catalog contract required by relay serving. pub async fn validate_serving_catalog(&self) -> Result<()> { - validate_serving_migration_version(self.live_migration_version().await?)?; - let runtime_columns = sqlx::query( "SELECT attname, format_type(atttypid, atttypmod) AS type_name, attnotnull \ FROM pg_attribute WHERE attrelid = 'communities'::regclass \ @@ -772,11 +758,9 @@ impl DeletionStore { /// Validate the exact live scoped-table and write-fence catalog for destruction. /// - /// Unlike relay serving compatibility, this intentionally rejects newer - /// migrations and unknown tenant tables until the deletion manifest changes. + /// Exact table and fence equality rejects unknown tenant data even + /// while unrelated SQLx migrations continue to advance. pub async fn validate_catalog(&self) -> Result<()> { - validate_destructive_migration_version(self.live_migration_version().await?)?; - let expected = EXPECTED_SCOPED_TABLES .iter() .copied() @@ -825,8 +809,6 @@ impl DeletionStore { let fenced_tables = self.live_fenced_tables().await?; let _ = community; // counts are intentionally not approval-bound for a live tenant. Ok(SchemaManifest { - revision: CATALOG_REVISION, - migration_version: EXPECTED_MIGRATION_VERSION, scoped_tables: live_tables.into_iter().collect(), row_counts: BTreeMap::new(), fenced_tables: fenced_tables.into_iter().collect(), @@ -2104,22 +2086,6 @@ impl DeletionStore { Ok(()) } - async fn live_migration_version(&self) -> Result> { - let ledger_exists: bool = - sqlx::query_scalar("SELECT to_regclass('_sqlx_migrations') IS NOT NULL") - .fetch_one(&self.pool) - .await?; - if !ledger_exists { - // Desired-state `pgschema` applies the checked-in final catalog but - // intentionally does not synthesize SQLx migration history. - return Ok(None); - } - sqlx::query_scalar("SELECT max(version) FROM _sqlx_migrations WHERE success") - .fetch_one(&self.pool) - .await - .map_err(Into::into) - } - async fn live_scoped_tables(&self) -> Result> { let rows: Vec = sqlx::query_scalar( r#" @@ -2168,28 +2134,6 @@ impl DeletionStore { } } -fn validate_destructive_migration_version(migration_version: Option) -> Result<()> { - if migration_version.is_some_and(|version| version != EXPECTED_MIGRATION_VERSION) { - Err(DbError::DeletionSafety(format!( - "community deletion schema migration drift: expected {EXPECTED_MIGRATION_VERSION}, got {}", - migration_version.expect("checked Some") - ))) - } else { - Ok(()) - } -} - -fn validate_serving_migration_version(migration_version: Option) -> Result<()> { - if migration_version.is_some_and(|version| version < EXPECTED_MIGRATION_VERSION) { - Err(DbError::DeletionSafety(format!( - "community serving fence migration is too old: require at least {EXPECTED_MIGRATION_VERSION}, got {}", - migration_version.expect("checked Some") - ))) - } else { - Ok(()) - } -} - /// Fail closed when a community-prefix inventory has an unsafe shape. pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> { if manifest.version != 4 { @@ -2608,19 +2552,6 @@ mod tests { ))); } - #[test] - fn migration_checks_accept_desired_state_catalogs_without_sqlx_ledger() { - assert!(validate_serving_migration_version(None).is_ok()); - assert!(validate_destructive_migration_version(None).is_ok()); - assert!(validate_serving_migration_version(Some(EXPECTED_MIGRATION_VERSION)).is_ok()); - assert!(validate_serving_migration_version(Some(EXPECTED_MIGRATION_VERSION + 1)).is_ok()); - assert!(validate_serving_migration_version(Some(EXPECTED_MIGRATION_VERSION - 1)).is_err()); - assert!(validate_destructive_migration_version(Some(EXPECTED_MIGRATION_VERSION)).is_ok()); - assert!( - validate_destructive_migration_version(Some(EXPECTED_MIGRATION_VERSION + 1)).is_err() - ); - } - #[test] fn storage_manifest_shape_invariants_fail_closed() { assert!(validate_storage_manifest(&storage_manifest()).is_ok()); @@ -2703,8 +2634,6 @@ mod tests { fn frozen_inventory_digest_is_stable() { let inventory = FrozenInventory { schema: SchemaManifest { - revision: 1, - migration_version: EXPECTED_MIGRATION_VERSION, scoped_tables: vec!["events".to_string()], row_counts: BTreeMap::from([("events".to_string(), 3)]), fenced_tables: vec!["events".to_string()], diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index c275e2baf1..49076a3684 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -1493,7 +1493,7 @@ mod tests { let mut digest_tampered = claim.request.clone(); digest_tampered.inventory_manifest = Some(serde_json::json!({ - "schema": {"revision": 0, "migration_version": 0, "scoped_tables": [], "row_counts": {}, "fenced_tables": []}, + "schema": {"scoped_tables": [], "row_counts": {}, "fenced_tables": []}, "storage": {"version": 4, "prefixes": []} })); assert!(validate_frozen_inventory(&digest_tampered).is_err()); From 7c6db47955a2481327f08f13225ddb5b206017aa Mon Sep 17 00:00:00 2001 From: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Wed, 5 Aug 2026 10:18:15 -0700 Subject: [PATCH 29/51] fix: preserve admitted leases through quiescing Co-authored-by: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/deletion.rs | 39 ++++++----- crates/buzz-deletion/src/lib.rs | 111 +++++++++++++++++++++++++------- 2 files changed, 110 insertions(+), 40 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index 4ba2d98517..15b2df568d 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -1089,7 +1089,8 @@ impl DeletionStore { /// /// The transition takes the same exclusive advisory lock as serving lease /// acquisition, so after commit no newer external effect can be admitted. - /// Already-acquired leases remain verifiable/releasable but cannot renew. + /// Already-acquired leases remain renewable, verifiable, and releasable so + /// admitted remote effects retain their exclusion proof until completion. pub async fn begin_quiescing(&self, token: &LeaseToken) -> Result<()> { let mut tx = self.pool.begin().await?; verify_lease(&mut tx, token, DeletionStage::Approved).await?; @@ -1921,11 +1922,14 @@ impl DeletionStore { Ok(lease) } - /// Renew an external side-effect lease only while the community is active. + /// Renew an already-admitted external side-effect lease while the community + /// is active or quiescing. /// - /// Quiescing rejects new acquisition and renewal. A pre-quiesce caller may - /// still verify/release its unexpired lease, but a long operation is - /// cancelled when its next heartbeat observes quiescing. + /// Quiescing rejects new acquisition, but the exact existing, unexpired + /// lease must remain renewable until its operation finishes. Otherwise the + /// heartbeat would abandon the exclusion proof while remote I/O may still + /// commit. Fence generation, owner, generation, expiry, and tombstone checks + /// continue to reject stale or post-fence renewal. pub async fn renew_serving_write_lease( &self, lease: &mut ServingWriteLease, @@ -1945,7 +1949,7 @@ impl DeletionStore { AND lease.generation = $4 AND lease.fence_generation = $5 \ AND lease.lease_until >= now() \ AND community.id = lease.community_id \ - AND community.deletion_state = 'active' \ + AND community.deletion_state IN ('active', 'quiescing') \ AND community.deleted_at IS NULL \ AND community.deletion_fence_generation = lease.fence_generation \ RETURNING lease.lease_until", @@ -1986,8 +1990,9 @@ impl DeletionStore { /// Check that an external side-effect lease remains current for finalization. /// - /// A lease admitted before quiescing may complete/release, but cannot renew; - /// this preserves an accurate bounded drain without admitting new work. + /// A lease admitted before quiescing may renew, complete, and release; new + /// work remains blocked, preserving an accurate drain without abandoning an + /// admitted remote effect. pub async fn verify_serving_write_lease(&self, lease: &ServingWriteLease) -> Result<()> { let mut tx = self.pool.begin().await?; sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") @@ -2999,7 +3004,7 @@ mod postgres_tests { #[tokio::test] #[ignore = "requires Postgres"] - async fn quiescing_rejects_new_and_renewed_leases_before_fence() { + async fn quiescing_rejects_new_leases_but_renews_admitted_lease_until_release() { let (db, store) = store().await; let (request, _) = inventoried_request(&db, &store).await; store @@ -3037,12 +3042,16 @@ mod postgres_tests { Err(DbError::AccessDenied(_)) )); assert!(store.verify_serving_write_lease(&serving).await.is_ok()); - assert!(matches!( - store - .renew_serving_write_lease(&mut serving, DEFAULT_LEASE_DURATION) - .await, - Err(DbError::AccessDenied(_)) - )); + let lease_until_before_renewal = serving.lease_until; + tokio::time::sleep(Duration::from_millis(10)).await; + store + .renew_serving_write_lease(&mut serving, DEFAULT_LEASE_DURATION) + .await + .expect("admitted lease renews while quiescing"); + assert!( + serving.lease_until > lease_until_before_renewal, + "renewal must extend the admitted lease" + ); assert!(matches!( store.fence(&claim.lease).await, Err(DbError::ServingWritesNotDrained { diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 49076a3684..9afd9a72dd 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -3,7 +3,7 @@ //! Shared durable whole-community deletion engine and store adapters. #[cfg(test)] -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -30,17 +30,7 @@ const LIST_PAGE_SIZE: usize = 1000; const RETRY_DELAY: Duration = Duration::from_secs(30); const HEARTBEAT_INTERVAL: Duration = Duration::from_secs(10); -#[cfg(test)] -static TEST_HEARTBEAT_INTERVAL_MS: AtomicU64 = AtomicU64::new(0); - fn heartbeat_interval() -> Duration { - #[cfg(test)] - { - let milliseconds = TEST_HEARTBEAT_INTERVAL_MS.load(Ordering::Relaxed); - if milliseconds > 0 { - return Duration::from_millis(milliseconds); - } - } HEARTBEAT_INTERVAL } @@ -171,6 +161,15 @@ pub async fn acquire_serving_write( db: &Db, community: buzz_core::CommunityId, operation: &str, +) -> Result { + acquire_serving_write_with_heartbeat(db, community, operation, heartbeat_interval()).await +} + +async fn acquire_serving_write_with_heartbeat( + db: &Db, + community: buzz_core::CommunityId, + operation: &str, + heartbeat_period: Duration, ) -> Result { let store = store(db); let owner = default_executor_id(); @@ -184,7 +183,7 @@ pub async fn acquire_serving_write( let lost = CancellationToken::new(); let heartbeat_lost = lost.clone(); tokio::spawn(async move { - let mut interval = tokio::time::interval(heartbeat_interval()); + let mut interval = tokio::time::interval(heartbeat_period); interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); interval.tick().await; loop { @@ -1382,7 +1381,7 @@ mod tests { } } - async fn claimed_test_deletion(prefix: &str) -> (Services, ClaimedDeletion) { + async fn claimed_test_deletion(prefix: &str) -> (Db, Services, ClaimedDeletion) { let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) .expect("BUZZ_TEST_DATABASE_URL or DATABASE_URL is required"); @@ -1446,7 +1445,7 @@ mod tests { .create_pool(Some(deadpool_redis::Runtime::Tokio1)) .expect("construct unused Redis pool"), }; - (services, claim) + (db, services, claim) } fn deletion_test_media_storage() -> Arc { @@ -1488,7 +1487,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] async fn frozen_inventory_digest_and_storage_ownership_fail_closed() { - let (_, claim) = claimed_test_deletion("deletion-integrity").await; + let (_, _, claim) = claimed_test_deletion("deletion-integrity").await; assert!(validate_frozen_inventory(&claim.request).is_ok()); let mut digest_tampered = claim.request.clone(); @@ -1512,7 +1511,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres and S3-compatible storage"] async fn drained_stage_resumes_chunk_deleted_before_stamp() { - let (mut services, claim) = claimed_test_deletion("deletion-chunk-resume").await; + let (_, mut services, claim) = claimed_test_deletion("deletion-chunk-resume").await; services.media = deletion_test_media_storage(); let community = claim.request.community_id; let meta_prefix = format!("_meta/{community}/"); @@ -1651,7 +1650,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres and S3-compatible storage"] async fn final_storage_verification_rejects_late_target_binding() { - let (mut services, claim) = claimed_test_deletion("deletion-late-binding").await; + let (_, mut services, claim) = claimed_test_deletion("deletion-late-binding").await; services.media = deletion_test_media_storage(); let community = claim.request.community_id; let late_key = format!("_meta/{community}/{}.json", "a".repeat(64)); @@ -1677,7 +1676,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] async fn stale_lease_during_failure_recording_is_lost_ownership() { - let (services, claim) = claimed_test_deletion("deletion-stale-record").await; + let (_, services, claim) = claimed_test_deletion("deletion-stale-record").await; let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") .or_else(|_| std::env::var("DATABASE_URL")) .expect("test database URL"); @@ -1702,6 +1701,65 @@ mod tests { assert!(!recorded); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_guard_heartbeats_long_operation_through_quiescing() { + let (db, services, claim) = claimed_test_deletion("serving-guard-quiesce").await; + let community = claim.request.community_id; + let guard = acquire_serving_write_with_heartbeat( + &db, + community, + "test_quiesce", + Duration::from_millis(10), + ) + .await + .expect("serving guard"); + + services + .store + .begin_quiescing(&claim.lease) + .await + .expect("quiesce"); + assert!( + services + .store + .acquire_serving_write_lease( + community, + "late_external", + "late-owner", + DEFAULT_LEASE_DURATION, + ) + .await + .is_err(), + "quiescing must reject newly admitted work" + ); + + let completed = Arc::new(AtomicBool::new(false)); + let operation_completed = Arc::clone(&completed); + let result = guard + .protect(async move { + tokio::time::sleep(Duration::from_millis(75)).await; + operation_completed.store(true, Ordering::Relaxed); + }) + .await; + assert!( + !guard.lost().is_cancelled(), + "quiescing must not cancel the admitted lease heartbeat" + ); + result.expect("admitted operation survives quiescing heartbeats"); + assert!(completed.load(Ordering::Relaxed)); + assert!(matches!( + services.store.fence(&claim.lease).await, + Err(buzz_db::DbError::ServingWritesNotDrained { + active_count: 1, + .. + }) + )); + + guard.finish().await.expect("release serving guard"); + assert_eq!(services.store.fence(&claim.lease).await.expect("fence"), 1); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn serving_guard_cancels_protected_operation_when_heartbeat_is_lost() { @@ -1721,10 +1779,14 @@ mod tests { .await .expect("create test community") .id; - TEST_HEARTBEAT_INTERVAL_MS.store(10, Ordering::Relaxed); - let guard = acquire_serving_write(&db, community, "test_cancel") - .await - .expect("serving guard"); + let guard = acquire_serving_write_with_heartbeat( + &db, + community, + "test_cancel", + Duration::from_millis(10), + ) + .await + .expect("serving guard"); sqlx::query("DELETE FROM community_serving_write_leases WHERE community_id = $1") .bind(community.as_uuid()) .execute(&pool) @@ -1738,7 +1800,6 @@ mod tests { operation_completed.store(true, Ordering::Relaxed); }) .await; - TEST_HEARTBEAT_INTERVAL_MS.store(0, Ordering::Relaxed); assert!(result.is_err(), "lease loss must reject the operation"); assert!( !completed.load(Ordering::Relaxed), @@ -1749,7 +1810,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] async fn guarded_external_step_rejects_preexisting_heartbeat_loss_without_polling_operation() { - let (services, claim) = claimed_test_deletion("deletion-heartbeat").await; + let (_, services, claim) = claimed_test_deletion("deletion-heartbeat").await; let heartbeat_lost = CancellationToken::new(); heartbeat_lost.cancel(); let polled = Arc::new(AtomicBool::new(false)); @@ -1782,7 +1843,7 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] async fn shutdown_during_stage_releases_claim_without_recording_retry() { - let (services, claim) = claimed_test_deletion("deletion-shutdown").await; + let (_, services, claim) = claimed_test_deletion("deletion-shutdown").await; let request_id = claim.request.id; let retry_count = claim.request.retry_count; let shutdown = CancellationToken::new(); From cc96b08f13a9f0a5348d6afefc366fe821ad9d3e Mon Sep 17 00:00:00 2001 From: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Wed, 5 Aug 2026 10:18:22 -0700 Subject: [PATCH 30/51] fix: return typed invite fence errors Co-authored-by: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/relay_invite.rs | 98 +++++++++++++++++++++++++++- crates/buzz-relay/src/api/invites.rs | 33 ++++++++-- 2 files changed, 124 insertions(+), 7 deletions(-) diff --git a/crates/buzz-db/src/relay_invite.rs b/crates/buzz-db/src/relay_invite.rs index 82b71b07bb..752797e2eb 100644 --- a/crates/buzz-db/src/relay_invite.rs +++ b/crates/buzz-db/src/relay_invite.rs @@ -111,6 +111,14 @@ pub async fn mint_relay_invite( let now = Utc::now(); let expires_at = now + chrono::Duration::seconds(ttl_secs as i64); + // Mint a v2 opaque invite inside the same lifecycle gate as every other + // community-scoped database write. The trigger remains the final backstop, + // but this typed guard keeps a quiescing community from surfacing as an + // opaque SQLSTATE/HTTP 500 at the API boundary. + let mut tx = pool.begin().await?; + crate::deletion::DeletionStore::new(pool.clone()) + .guard_transaction(&mut tx, community) + .await?; let row = sqlx::query( "INSERT INTO relay_invites (community_id, token_hash, max_uses, expires_at, created_by) \ VALUES ($1, $2, $3, $4, $5) \ @@ -121,8 +129,9 @@ pub async fn mint_relay_invite( .bind(max_uses) .bind(expires_at) .bind(created_by) - .fetch_one(pool) + .fetch_one(&mut *tx) .await?; + tx.commit().await?; let invite_id: uuid::Uuid = row.try_get("id")?; @@ -374,6 +383,7 @@ pub async fn claim_relay_invite( mod tests { use super::*; use crate::relay_members::is_relay_member; + use sha2::Digest; use sqlx::PgPool; use uuid::Uuid; @@ -448,6 +458,92 @@ mod tests { } } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn mint_after_quiescing_returns_typed_fence_without_persisting() { + let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| TEST_DB_URL.to_owned()); + let db = crate::Db::new(&crate::DbConfig { + database_url, + max_connections: 5, + min_connections: 0, + ..crate::DbConfig::default() + }) + .await + .expect("connect invite deletion test DB"); + db.migrate().await.expect("migrate invite deletion test DB"); + let pool = db.pool.clone(); + let store = db.deletion_store(); + let host = format!("relay-invite-fence-{}.example", Uuid::new_v4().simple()); + let community = db + .ensure_configured_community(&host) + .await + .expect("create fenced invite community") + .id; + let request = store + .submit(&host, "owner", None) + .await + .expect("submit deletion request"); + let empty_digest = hex::encode(sha2::Sha256::digest([])); + let inventory = crate::deletion::FrozenInventory { + schema: store + .inventory_schema(community) + .await + .expect("inventory schema"), + storage: crate::deletion::StorageManifest { + version: 4, + prefixes: [ + format!("_meta/{community}/"), + format!("_uploads/{community}/"), + format!("repos/{community}/"), + ] + .into_iter() + .map(|prefix| crate::deletion::PrefixManifest { + prefix, + object_count: 0, + total_bytes: 0, + keys_digest: empty_digest.clone(), + }) + .collect(), + }, + }; + store + .freeze_inventory(request.id, &inventory) + .await + .expect("freeze inventory"); + store + .approve(request.id, "owner", None) + .await + .expect("approve deletion"); + let claim = store + .claim_specific( + request.id, + "executor", + crate::deletion::DEFAULT_LEASE_DURATION, + ) + .await + .expect("claim deletion") + .expect("runnable deletion"); + store + .begin_quiescing(&claim.lease) + .await + .expect("begin quiescing"); + + let error = mint_relay_invite(&pool, community, "owner", 3600, Some(1)) + .await + .expect_err("quiescing must reject invite minting"); + assert!(matches!(error, crate::error::DbError::AccessDenied(_))); + + let invite_count: i64 = + sqlx::query_scalar("SELECT count(*) FROM relay_invites WHERE community_id = $1") + .bind(community.as_uuid()) + .fetch_one(&pool) + .await + .expect("count relay invites"); + assert_eq!(invite_count, 0, "rejected mint must not persist an invite"); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn bounded_claim_exhausts_and_existing_member_retry_does_not_consume() { diff --git a/crates/buzz-relay/src/api/invites.rs b/crates/buzz-relay/src/api/invites.rs index 60b7caf2af..d09c7fc611 100644 --- a/crates/buzz-relay/src/api/invites.rs +++ b/crates/buzz-relay/src/api/invites.rs @@ -260,6 +260,19 @@ async fn authenticate( Ok((tenant, pubkey)) } +fn map_mint_error(error: buzz_db::DbError) -> (StatusCode, Json) { + match error { + buzz_db::DbError::InvalidData(message) | buzz_db::DbError::DeletionSafety(message) => { + api_error(StatusCode::BAD_REQUEST, &message) + } + buzz_db::DbError::AccessDenied(_) => api_error( + StatusCode::SERVICE_UNAVAILABLE, + "community writes are temporarily unavailable", + ), + error => internal_error(&format!("invite mint: {error}")), + } +} + /// Mint an invite code — `POST /api/invites`, NIP-98 signed by an owner/admin. /// /// Returns the code, its expiry, and a shareable landing-page URL on the @@ -304,12 +317,7 @@ pub async fn mint_invite( .db .mint_relay_invite(tenant.community(), &sender_hex, ttl, max_uses) .await - .map_err(|error| match error { - buzz_db::DbError::InvalidData(message) | buzz_db::DbError::DeletionSafety(message) => { - api_error(StatusCode::BAD_REQUEST, &message) - } - error => internal_error(&format!("invite mint: {error}")), - })?; + .map_err(map_mint_error)?; // Same TLS-posture logic as nip98_expected_url: wss deployments get an // https landing page URL, ws dev/test deployments get http. @@ -897,6 +905,19 @@ mod tests { } } + #[test] + fn mint_fence_errors_map_to_temporary_unavailability() { + let (status, body) = super::map_mint_error(buzz_db::DbError::AccessDenied( + "community is write-fenced".to_string(), + )); + + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + body.0.get("error").and_then(Value::as_str), + Some("community writes are temporarily unavailable") + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn malformed_and_unknown_v2_codes_are_forbidden_without_v1_fallback() { From d922b427fa1eb84a3acbd89bfaee63a415316c23 Mon Sep 17 00:00:00 2001 From: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Date: Wed, 5 Aug 2026 12:33:54 -0700 Subject: [PATCH 31/51] test: isolate quiesced invite migration database Co-authored-by: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> Signed-off-by: am <6e30cd56c30e030cd31bb0939b94a7c257c9a09d5ba2d92cf2735da45629f248@buzz.block.builderlab.xyz> --- crates/buzz-db/src/relay_invite.rs | 49 +++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/crates/buzz-db/src/relay_invite.rs b/crates/buzz-db/src/relay_invite.rs index 752797e2eb..3f78c7c6e1 100644 --- a/crates/buzz-db/src/relay_invite.rs +++ b/crates/buzz-db/src/relay_invite.rs @@ -390,14 +390,46 @@ mod tests { const TEST_DB_URL: &str = "postgres://buzz:buzz_dev@localhost:5432/buzz"; // sadscan:disable np.postgres.1 async fn setup_pool() -> PgPool { - let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") - .or_else(|_| std::env::var("DATABASE_URL")) - .unwrap_or_else(|_| TEST_DB_URL.to_owned()); - PgPool::connect(&database_url) + PgPool::connect(&test_database_url()) .await .expect("connect to test DB") } + fn test_database_url() -> String { + std::env::var("BUZZ_TEST_DATABASE_URL") + .or_else(|_| std::env::var("DATABASE_URL")) + .unwrap_or_else(|_| TEST_DB_URL.to_owned()) + } + + async fn create_scratch_database(prefix: &str) -> (PgPool, String, String) { + let admin_url = test_database_url(); + let admin = PgPool::connect(&admin_url) + .await + .expect("connect to test database server"); + let name = format!("{}_{}", prefix, Uuid::new_v4().simple()); + sqlx::query(sqlx::AssertSqlSafe(format!("CREATE DATABASE {name}"))) + .execute(&admin) + .await + .expect("create scratch database"); + let path_start = admin_url + .rfind('/') + .expect("database URL has a path segment"); + let scratch_url = format!("{}/{}", &admin_url[..path_start], name); + (admin, name, scratch_url) + } + + async fn drop_scratch_database(admin: PgPool, db: crate::Db, name: &str) { + db.pool.close().await; + drop(db); + sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE IF EXISTS {name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop scratch database"); + admin.close().await; + } + async fn make_test_community(pool: &PgPool) -> CommunityId { let id = Uuid::new_v4(); sqlx::query("INSERT INTO communities (id, host) VALUES ($1, $2)") @@ -461,9 +493,8 @@ mod tests { #[tokio::test] #[ignore = "requires Postgres"] async fn mint_after_quiescing_returns_typed_fence_without_persisting() { - let database_url = std::env::var("BUZZ_TEST_DATABASE_URL") - .or_else(|_| std::env::var("DATABASE_URL")) - .unwrap_or_else(|_| TEST_DB_URL.to_owned()); + let (admin, database_name, database_url) = + create_scratch_database("relay_invite_fence").await; let db = crate::Db::new(&crate::DbConfig { database_url, max_connections: 5, @@ -542,6 +573,10 @@ mod tests { .await .expect("count relay invites"); assert_eq!(invite_count, 0, "rejected mint must not persist an invite"); + + drop(store); + drop(pool); + drop_scratch_database(admin, db, &database_name).await; } #[tokio::test] From 77fcb86c4caf59d983d53f39321910c6191cd44e Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 19:05:34 -0700 Subject: [PATCH 32/51] fix: isolate community deletion fences Co-authored-by: cid Signed-off-by: cid --- Justfile | 6 +- bin/.ast-grep-0.39.7.pkg | 1 + bin/ast-grep | 1 + crates/buzz-db/src/channel.rs | 1 + crates/buzz-db/src/deletion.rs | 299 +++++++++++++++++++++- crates/buzz-db/src/lib.rs | 3 +- crates/buzz-db/src/migration.rs | 4 + crates/buzz-db/src/push.rs | 126 ++++++++- crates/buzz-db/src/relay_invite.rs | 68 +++++ crates/buzz-db/src/workflow.rs | 1 + crates/buzz-deletion/src/lib.rs | 34 ++- crates/buzz-relay/src/handlers/event.rs | 102 +++----- migrations/0028_community_deletion.sql | 35 +++ schema/schema.sql | 32 +++ scripts/check-community-fenced-writes.py | 208 +++++++++++++++ scripts/lints/community_fenced_writes.yml | 11 + 16 files changed, 847 insertions(+), 85 deletions(-) create mode 120000 bin/.ast-grep-0.39.7.pkg create mode 120000 bin/ast-grep create mode 100755 scripts/check-community-fenced-writes.py create mode 100644 scripts/lints/community_fenced_writes.yml diff --git a/Justfile b/Justfile index 0a43249d5f..6aac156adf 100644 --- a/Justfile +++ b/Justfile @@ -92,7 +92,11 @@ build-release: cargo build --workspace --release # Run repo lint and formatting checks -check: fmt-check clippy desktop-check desktop-tauri-fmt-check desktop-tauri-clippy web-check mobile-check +check: fmt-check community-fenced-writes-check clippy desktop-check desktop-tauri-fmt-check desktop-tauri-clippy web-check mobile-check + +# Structurally classify production SQL writes to community-fenced tables. +community-fenced-writes-check: + ./scripts/check-community-fenced-writes.py # Format all Rust code fmt: diff --git a/bin/.ast-grep-0.39.7.pkg b/bin/.ast-grep-0.39.7.pkg new file mode 120000 index 0000000000..383f4511d4 --- /dev/null +++ b/bin/.ast-grep-0.39.7.pkg @@ -0,0 +1 @@ +hermit \ No newline at end of file diff --git a/bin/ast-grep b/bin/ast-grep new file mode 120000 index 0000000000..dd35c6a667 --- /dev/null +++ b/bin/ast-grep @@ -0,0 +1 @@ +.ast-grep-0.39.7.pkg \ No newline at end of file diff --git a/crates/buzz-db/src/channel.rs b/crates/buzz-db/src/channel.rs index 9d15fccfc8..c8fd6343e3 100644 --- a/crates/buzz-db/src/channel.rs +++ b/crates/buzz-db/src/channel.rs @@ -1508,6 +1508,7 @@ pub async fn reap_expired_ephemeral_channels(pool: &PgPool) -> Result Result { self.validate_catalog().await?; let live_tables = self.live_scoped_tables().await?; let fenced_tables = self.live_fenced_tables().await?; - let _ = community; // counts are intentionally not approval-bound for a live tenant. + let mut row_counts = BTreeMap::new(); + for table in &live_tables { + let sql = format!("SELECT count(*)::BIGINT FROM {table} WHERE community_id = $1"); + let count: i64 = sqlx::query_scalar(AssertSqlSafe(sql)) + .bind(community.as_uuid()) + .fetch_one(&self.pool) + .await?; + row_counts.insert(table.clone(), count); + } Ok(SchemaManifest { scoped_tables: live_tables.into_iter().collect(), - row_counts: BTreeMap::new(), + row_counts, fenced_tables: fenced_tables.into_iter().collect(), }) } @@ -1764,6 +1780,87 @@ impl DeletionStore { Ok(()) } + /// Clear a fail-closed block after an operator has remediated its cause. + /// + /// Recovery preserves the immutable target, approval, inventory, stage, + /// fence generation, and prior failure checkpoint. It appends an auditable + /// operator checkpoint and only makes runnable stages immediately claimable. + pub async fn unblock( + &self, + request_id: Uuid, + unblocked_by: &str, + reason: &str, + ) -> Result { + let unblocked_by = unblocked_by.trim(); + let reason = reason.trim(); + if unblocked_by.is_empty() || reason.is_empty() { + return Err(DbError::InvalidData( + "unblock requires non-empty operator identity and remediation reason".to_string(), + )); + } + + let mut tx = self.pool.begin().await?; + let request_row = sqlx::query( + "SELECT * FROM community_deletion_requests \ + WHERE id = $1 AND blocked_at IS NOT NULL FOR UPDATE", + ) + .bind(request_id) + .fetch_optional(&mut *tx) + .await? + .ok_or_else(|| { + DbError::DeletionSafety(format!( + "deletion {request_id} is missing or is not blocked" + )) + })?; + let request = row_to_request(request_row)?; + if !request.stage.runnable() { + return Err(DbError::DeletionSafety(format!( + "blocked deletion {request_id} at stage {} cannot resume", + request.stage + ))); + } + if request.lease_owner.is_some() + && request + .lease_until + .is_some_and(|lease_until| lease_until >= Utc::now()) + { + return Err(DbError::DeletionSafety(format!( + "blocked deletion {request_id} still has a live executor lease" + ))); + } + + let prior_block = request.blocked_reason.clone(); + sqlx::query( + "INSERT INTO community_deletion_checkpoints \ + (request_id, stage, unit_key, status, lease_generation, detail, completed_at) \ + VALUES ($1, $2, $3, 'completed', $4, $5, now())", + ) + .bind(request.id) + .bind(request.stage.to_string()) + .bind(format!("operator_unblock:{}", Uuid::new_v4())) + .bind(request.lease_generation.max(1)) + .bind(serde_json::json!({ + "unblocked_by": bound_text(unblocked_by, 512), + "reason": bound_text(reason, 4096), + "previous_block": prior_block, + })) + .execute(&mut *tx) + .await?; + + let row = sqlx::query( + "UPDATE community_deletion_requests \ + SET blocked_at = NULL, blocked_reason = NULL, retry_count = 0, \ + last_error = NULL, last_error_at = NULL, next_attempt_at = now(), \ + lease_owner = NULL, lease_until = NULL, updated_at = now() \ + WHERE id = $1 AND blocked_at IS NOT NULL RETURNING *", + ) + .bind(request_id) + .fetch_one(&mut *tx) + .await?; + tx.commit().await?; + row_to_request(row) + } + /// Persist a fail-closed permanent block and release the claim. pub async fn block( &self, @@ -1887,10 +1984,24 @@ impl DeletionStore { ) -> Result { let lease_seconds = i64::try_from(lease_duration.as_secs()).unwrap_or(i64::MAX); let mut tx = self.pool.begin().await?; - sqlx::query("SELECT pg_advisory_xact_lock_shared(community_deletion_lock_key($1))") + // The assertion owns both the shared ordering lock and the supported + // READ COMMITTED check. The lease table is trigger-excluded, so this + // explicit admission is its database-enforced write fence. + if let Err(error) = sqlx::query("SELECT assert_community_write_allowed($1)") .bind(community.as_uuid()) .execute(&mut *tx) - .await?; + .await + { + if error.as_database_error().is_some_and(|database_error| { + database_error.code().as_deref() == Some("55000") + && database_error.message().starts_with("community write") + }) { + return Err(DbError::AccessDenied(format!( + "community {community} is write-fenced or missing" + ))); + } + return Err(error.into()); + } let row = sqlx::query( "INSERT INTO community_serving_write_leases \ (community_id, operation, owner, fence_generation, lease_until) \ @@ -2740,9 +2851,9 @@ mod postgres_tests { .expect("claim before approval") .is_none()); - // Approval binds structural/storage taxonomy, not live row counts. A - // serving write between inventory and approval must not make an active - // community undeletable. + // Row counts are frozen observational evidence. Ordinary serving + // churn after inventory does not invalidate approval: execution fences, + // purges, and verifies the live tenant state independently. db.add_to_allowlist(request.community_id, &[7_u8; 32], &[8_u8; 32], None) .await .expect("post-inventory serving write"); @@ -2750,7 +2861,12 @@ mod postgres_tests { .inventory_schema(request.community_id) .await .expect("live schema after row churn"); - assert_eq!(current_schema, inventory.schema); + assert_eq!( + current_schema.row_counts["pubkey_allowlist"], + inventory.schema.row_counts["pubkey_allowlist"] + 1 + ); + assert_eq!(current_schema.scoped_tables, inventory.schema.scoped_tables); + assert_eq!(current_schema.fenced_tables, inventory.schema.fenced_tables); let mismatched_insert = sqlx::query( "INSERT INTO community_deletion_approvals \ @@ -2907,6 +3023,59 @@ mod postgres_tests { .is_none()); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn operator_unblock_preserves_approval_and_records_recovery() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + let claim = store + .claim_specific(request.id, "executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim") + .expect("approved request is claimable"); + store + .block( + &claim.lease, + DeletionStage::Approved, + "dependency", + "operator repair required", + ) + .await + .expect("block request"); + + assert!(store.unblock(request.id, "", "repair").await.is_err()); + let recovered = store + .unblock(request.id, "operator", "bucket policy repaired") + .await + .expect("unblock after remediation"); + assert_eq!(recovered.stage, DeletionStage::Approved); + assert!(recovered.blocked_reason.is_none()); + assert!(recovered.last_error.is_none()); + assert_eq!(recovered.inventory_digest, request.inventory_digest); + assert_eq!(recovered.fence_generation, request.fence_generation); + assert!(store + .unblock(request.id, "operator", "again") + .await + .is_err()); + assert!(store + .claim_specific(request.id, "successor", DEFAULT_LEASE_DURATION) + .await + .expect("claim recovered request") + .is_some()); + + let inspection = store.inspect(request.id).await.expect("inspect recovery"); + assert!(inspection.checkpoints.iter().any(|checkpoint| { + checkpoint.unit_key.starts_with("operator_unblock:") + && checkpoint.detail["unblocked_by"] == "operator" + && checkpoint.detail["reason"] == "bucket policy repaired" + && checkpoint.detail["previous_block"] == "operator repair required" + })); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn stale_claim_and_fence_generation_fail_closed() { @@ -3002,6 +3171,50 @@ mod postgres_tests { ); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn write_assertion_rejects_pinned_snapshot_isolation_before_authorization() { + let (db, _) = store().await; + let community = db + .ensure_configured_community(&format!( + "isolation-guard-{}.example", + Uuid::new_v4().simple() + )) + .await + .expect("create community") + .id; + + for isolation in ["REPEATABLE READ", "SERIALIZABLE"] { + let mut tx = db.pool.begin().await.expect("begin isolation probe"); + sqlx::query(sqlx::AssertSqlSafe(format!( + "SET TRANSACTION ISOLATION LEVEL {isolation}" + ))) + .execute(&mut *tx) + .await + .expect("set transaction isolation"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(community.to_string()) + .execute(&mut *tx) + .await + .expect("forge executor authorization"); + let error = sqlx::query("SELECT assert_community_write_allowed($1)") + .bind(community.as_uuid()) + .execute(&mut *tx) + .await + .expect_err("pinned snapshot isolation must fail before authorization"); + assert_eq!( + error + .as_database_error() + .and_then(|error| error.code()) + .as_deref(), + Some("25000") + ); + } + } + #[tokio::test] #[ignore = "requires Postgres"] async fn quiescing_rejects_new_leases_but_renews_admitted_lease_until_release() { @@ -3131,6 +3344,76 @@ mod postgres_tests { assert_eq!(after.expired, before.expired - 2); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn serving_lease_reaper_remains_global_across_tombstoned_tenant() { + let (db, store) = store().await; + let active_a = db + .ensure_configured_community(&format!("lease-a-{}.example", Uuid::new_v4().simple())) + .await + .expect("active A") + .id; + let target = db + .ensure_configured_community(&format!("lease-t-{}.example", Uuid::new_v4().simple())) + .await + .expect("target T") + .id; + let active_x = db + .ensure_configured_community(&format!("lease-x-{}.example", Uuid::new_v4().simple())) + .await + .expect("active X") + .id; + store + .reap_expired_serving_write_leases(10_000) + .await + .expect("clear unrelated expired leases"); + for (community, owner) in [(active_a, "a"), (target, "t"), (active_x, "x")] { + let lease = store + .acquire_serving_write_lease( + community, + "global_reaper_test", + owner, + DEFAULT_LEASE_DURATION, + ) + .await + .expect("acquire lease"); + sqlx::query( + "UPDATE community_serving_write_leases \ + SET lease_until = now() - interval '1 second' WHERE id = $1", + ) + .bind(lease.id) + .execute(&db.pool) + .await + .expect("expire lease"); + } + let mut lifecycle = db.pool.begin().await.expect("begin target lifecycle"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '1', true)", + ) + .bind(target.to_string()) + .execute(&mut *lifecycle) + .await + .expect("authorize tombstone fixture"); + sqlx::query( + "UPDATE communities SET deletion_state = 'tombstone', \ + deletion_fence_generation = 1, deleted_at = now() WHERE id = $1", + ) + .bind(target.as_uuid()) + .execute(&mut *lifecycle) + .await + .expect("tombstone target"); + lifecycle.commit().await.expect("commit tombstone fixture"); + + assert_eq!( + store + .reap_expired_serving_write_leases(10) + .await + .expect("global lease reap"), + 3 + ); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn checkpointed_resume_is_idempotent_and_tombstone_blocks_name_reuse() { diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 92a2ac942b..3e1864fb7c 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -3987,7 +3987,8 @@ impl Db { WHERE elem->>0 = 'd' LIMIT 1), \ '' \ ) \ - WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL", + WHERE kind BETWEEN 30000 AND 39999 AND d_tag IS NULL \ + AND community_write_allowed(community_id)", ) .execute(&self.pool) .await?; diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index af1bf8cb46..1d6f699d01 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -1093,7 +1093,11 @@ mod tests { assert!(deletion.contains("CREATE TABLE community_deletion_checkpoints")); assert!(deletion.contains("CREATE TABLE community_serving_write_leases")); assert!(deletion.contains("CREATE TABLE community_deletion_executor_heartbeats")); + assert!(deletion.contains("CREATE FUNCTION community_write_allowed")); + assert!(deletion.contains("LANGUAGE plpgsql VOLATILE")); assert!(deletion.contains("CREATE FUNCTION assert_community_write_allowed")); + assert!(deletion.contains("current_setting('transaction_isolation') <> 'read committed'")); + assert!(deletion.contains("ERRCODE = 'invalid_transaction_state'")); assert!(deletion.contains("CREATE FUNCTION enforce_community_write_fence")); assert!(deletion.contains("CREATE FUNCTION attach_community_write_fence")); assert!(deletion.contains("community_write_fence_excluded_table")); diff --git a/crates/buzz-db/src/push.rs b/crates/buzz-db/src/push.rs index 04b6a7ae36..0b3245ffcc 100644 --- a/crates/buzz-db/src/push.rs +++ b/crates/buzz-db/src/push.rs @@ -852,6 +852,7 @@ where WHERE attempts < $3 AND next_attempt_at <= now() AND (state = 'pending' OR (state = 'matching' AND lease_until < now())) + AND community_write_allowed(community_id) ORDER BY next_attempt_at, created_at LIMIT 1 ), @@ -933,7 +934,8 @@ where pub async fn reap_exhausted_matches(pool: &PgPool) -> Result { Ok(sqlx::query( "DELETE FROM push_match_queue WHERE attempts >= $1 \ - AND (state='pending' OR (state='matching' AND lease_until < now()))", + AND (state='pending' OR (state='matching' AND lease_until < now())) \ + AND community_write_allowed(community_id)", ) .bind(MAX_MATCH_ATTEMPTS) .execute(pool) @@ -2121,6 +2123,128 @@ mod tests { ); } + async fn seed_matcher_fixture( + pool: &PgPool, + community: CommunityId, + marker: u8, + attempts: i32, + age_seconds: i64, + ) { + let event_id = vec![marker; 32]; + sqlx::query( + "INSERT INTO events \ + (community_id, id, pubkey, created_at, kind, tags, content, sig) \ + VALUES ($1, $2, $3, to_timestamp(1), 9, '[]', '', $4)", + ) + .bind(community.as_uuid()) + .bind(&event_id) + .bind(vec![marker.saturating_add(20); 32]) + .bind(vec![marker.saturating_add(30); 64]) + .execute(pool) + .await + .expect("seed source event"); + sqlx::query( + "INSERT INTO push_match_queue \ + (community_id, event_id, attempts, next_attempt_at) \ + VALUES ($1, $2, $3, now() - make_interval(secs => $4))", + ) + .bind(community.as_uuid()) + .bind(event_id) + .bind(attempts) + .bind(age_seconds) + .execute(pool) + .await + .expect("seed matcher row"); + } + + async fn quiesce_test_community(pool: &PgPool, community: CommunityId) { + let mut lifecycle = pool.begin().await.expect("begin lifecycle fixture"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(community.to_string()) + .execute(&mut *lifecycle) + .await + .expect("authorize target lifecycle"); + sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1") + .bind(community.as_uuid()) + .execute(&mut *lifecycle) + .await + .expect("quiesce target"); + lifecycle.commit().await.expect("commit target lifecycle"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn matcher_claim_skips_quiescing_tenant_while_active_bystanders_progress() { + let pool = setup_pool().await; + sqlx::query("DELETE FROM push_match_queue WHERE community_write_allowed(community_id)") + .execute(&pool) + .await + .expect("drain active matcher queue"); + let active_a = make_community(&pool).await; + let target = make_community(&pool).await; + let active_x = make_community(&pool).await; + seed_matcher_fixture(&pool, active_a, 1, 0, 30).await; + seed_matcher_fixture(&pool, target, 2, 0, 40).await; + seed_matcher_fixture(&pool, active_x, 3, 0, 20).await; + quiesce_test_community(&pool, target).await; + + let batch = claim_due_match_batch(&pool, 16, Utc::now() + chrono::Duration::minutes(1)) + .await + .expect("claim active bystander") + .expect("active A is claimable despite older target row"); + assert_eq!(batch.community, active_a); + let target_state: String = + sqlx::query_scalar("SELECT state FROM push_match_queue WHERE community_id = $1") + .bind(target.as_uuid()) + .fetch_one(&pool) + .await + .expect("target row remains attributed"); + assert_eq!(target_state, "pending"); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn exhausted_match_reaper_skips_quiescing_tenant_and_reaps_active_bystanders() { + let pool = setup_pool().await; + sqlx::query("DELETE FROM push_match_queue WHERE community_write_allowed(community_id)") + .execute(&pool) + .await + .expect("drain active matcher queue"); + let active_a = make_community(&pool).await; + let target = make_community(&pool).await; + let active_x = make_community(&pool).await; + seed_matcher_fixture(&pool, active_a, 4, MAX_MATCH_ATTEMPTS, 30).await; + seed_matcher_fixture(&pool, target, 5, MAX_MATCH_ATTEMPTS, 40).await; + seed_matcher_fixture(&pool, active_x, 6, MAX_MATCH_ATTEMPTS, 20).await; + quiesce_test_community(&pool, target).await; + + assert_eq!( + reap_exhausted_matches(&pool) + .await + .expect("reap active bystanders"), + 2 + ); + let target_remaining: i64 = + sqlx::query_scalar("SELECT count(*) FROM push_match_queue WHERE community_id = $1") + .bind(target.as_uuid()) + .fetch_one(&pool) + .await + .expect("target exhausted row remains attributed"); + assert_eq!(target_remaining, 1); + for active in [active_a, active_x] { + let active_remaining: i64 = + sqlx::query_scalar("SELECT count(*) FROM push_match_queue WHERE community_id = $1") + .bind(active.as_uuid()) + .fetch_one(&pool) + .await + .expect("active bystander is drained"); + assert_eq!(active_remaining, 0); + } + } + /// T2b batch contract: one claim returns jobs from exactly ONE community /// (so downstream lease/membership loads are single statements), the /// set-wise complete and retry honor the claim fence, and a retried job diff --git a/crates/buzz-db/src/relay_invite.rs b/crates/buzz-db/src/relay_invite.rs index 3f78c7c6e1..14331b022f 100644 --- a/crates/buzz-db/src/relay_invite.rs +++ b/crates/buzz-db/src/relay_invite.rs @@ -176,6 +176,7 @@ pub async fn reap_expired_relay_invites(pool: &PgPool, cutoff: DateTime) -> WHERE (community_id, id) IN (\ SELECT community_id, id FROM relay_invites \ WHERE expires_at < $1 \ + AND community_write_allowed(community_id) \ ORDER BY expires_at \ LIMIT $2\ )", @@ -746,6 +747,73 @@ mod tests { delete_test_community(&pool, community).await; } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn retention_sweep_skips_quiescing_tenant_while_active_bystanders_progress() { + let (admin, database_name, database_url) = + create_scratch_database("relay_invite_liveness").await; + let db = crate::Db::new(&crate::DbConfig { + database_url, + max_connections: 5, + min_connections: 0, + ..crate::DbConfig::default() + }) + .await + .expect("connect invite liveness database"); + db.migrate() + .await + .expect("migrate invite liveness database"); + let pool = db.pool.clone(); + let active_a = make_test_community(&pool).await; + let target = make_test_community(&pool).await; + let active_x = make_test_community(&pool).await; + let cutoff = Utc::now(); + for community in [active_a, target, active_x] { + sqlx::query( + "INSERT INTO relay_invites \ + (community_id, token_hash, expires_at, created_by) \ + VALUES ($1, $2, $3, 'test')", + ) + .bind(community.as_uuid()) + .bind(sha2::Sha256::digest(community.as_uuid().as_bytes()).as_slice()) + .bind(cutoff - chrono::Duration::seconds(1)) + .execute(&pool) + .await + .expect("seed expired invite"); + } + let mut lifecycle = pool.begin().await.expect("begin lifecycle fixture"); + sqlx::query( + "SELECT set_config('buzz.deletion_executor_community', $1, true), \ + set_config('buzz.deletion_fence_generation', '0', true)", + ) + .bind(target.to_string()) + .execute(&mut *lifecycle) + .await + .expect("authorize lifecycle fixture"); + sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1") + .bind(target.as_uuid()) + .execute(&mut *lifecycle) + .await + .expect("quiesce target"); + lifecycle.commit().await.expect("commit lifecycle fixture"); + + assert_eq!( + reap_expired_relay_invites(&pool, cutoff) + .await + .expect("reap active bystanders"), + 2 + ); + let remaining: Vec = + sqlx::query_scalar("SELECT community_id FROM relay_invites ORDER BY community_id") + .fetch_all(&pool) + .await + .expect("read remaining invite attribution"); + assert_eq!(remaining, vec![*target.as_uuid()]); + + drop(pool); + drop_scratch_database(admin, db, &database_name).await; + } + #[tokio::test] #[ignore = "requires Postgres"] async fn unlimited_invites_count_each_new_member() { diff --git a/crates/buzz-db/src/workflow.rs b/crates/buzz-db/src/workflow.rs index 7a2396c1fd..ad1fd3a939 100644 --- a/crates/buzz-db/src/workflow.rs +++ b/crates/buzz-db/src/workflow.rs @@ -602,6 +602,7 @@ pub async fn prune_scheduled_workflow_fires_before( r#" DELETE FROM scheduled_workflow_fires WHERE claimed_at < $1 + AND community_write_allowed(community_id) "#, ) .bind(older_than) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 9afd9a72dd..2fa4dbaee3 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -251,6 +251,17 @@ pub enum Command { #[arg(long)] note: Option, }, + /// Resume a blocked request after remediating its recorded failure. + Unblock { + /// Deletion request UUID. + id: Uuid, + /// Operator identity recorded in the recovery checkpoint. + #[arg(long)] + unblocked_by: String, + /// Remediation or change reference recorded in the checkpoint. + #[arg(long)] + reason: String, + }, /// Claim and run one request until terminal/blocked. Run { /// Deletion request UUID. @@ -335,6 +346,8 @@ fn is_permanent_error(error: &anyhow::Error) -> bool { struct RunOutput { request_id: Uuid, stage: DeletionStage, + retry_count: i32, + last_error: Option, blocked_reason: Option, } @@ -360,6 +373,15 @@ pub async fn run(command: Command) -> Result { print_json(&store.approve(id, &approved_by, note.as_deref()).await?)?; Ok(0) } + Command::Unblock { + id, + unblocked_by, + reason, + } => { + let store = connect_store().await?; + print_json(&store.unblock(id, &unblocked_by, &reason).await?)?; + Ok(0) + } command => run_with_services(command, connect_services().await?).await, } } @@ -426,7 +448,10 @@ async fn run_with_services(command: Command, services: Services) -> Result print_json(&sweep)?; Ok(i32::from(sweep.unknown_object_count > 0)) } - Command::List { .. } | Command::Inspect { .. } | Command::Approve { .. } => { + Command::List { .. } + | Command::Inspect { .. } + | Command::Approve { .. } + | Command::Unblock { .. } => { anyhow::bail!("database-only command reached full-service dispatcher") } } @@ -741,8 +766,9 @@ async fn run_loop( ran = true; let output = execute_claim(&services, mode, claim, &shutdown).await?; print_json(&output)?; - if mode == LoopMode::Run || shutdown.is_cancelled() { - return Ok(i32::from(output.blocked_reason.is_some())); + let failed = output.last_error.is_some() || output.blocked_reason.is_some(); + if mode == LoopMode::Run || shutdown.is_cancelled() || failed { + return Ok(i32::from(failed)); } } } @@ -1321,6 +1347,8 @@ fn run_output(request: DeletionRequest) -> RunOutput { RunOutput { request_id: request.id, stage: request.stage, + retry_count: request.retry_count, + last_error: request.last_error, blocked_reason: request.blocked_reason, } } diff --git a/crates/buzz-relay/src/handlers/event.rs b/crates/buzz-relay/src/handlers/event.rs index 05865ece64..ccba40f328 100644 --- a/crates/buzz-relay/src/handlers/event.rs +++ b/crates/buzz-relay/src/handlers/event.rs @@ -705,53 +705,49 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc guard, - Err(error) => { + Ok(true) => {} + Ok(false) => { reject("restricted"); conn.send(RelayMessage::ok( &event_id_hex, false, - &format!("restricted: community writes are fenced: {error}"), + "restricted: community writes are fenced", )); return; } - }; - let handled = serving_write - .protect(handle_ephemeral_event( - event, - conn_id, - pubkey_bytes, - auth_pubkey, - Arc::clone(&conn), - state, - )) - .await; - let terminal = match handled { - Ok(Ok(())) => EphemeralTerminal::Accepted, - Ok(Err(message)) => { - reject("invalid"); - EphemeralTerminal::Rejected(message) - } Err(error) => { - reject("restricted"); - EphemeralTerminal::LeaseLost(format!( - "restricted: community write lease lost: {error}" - )) + reject("error"); + tracing::warn!(%error, event_id = %event_id_hex, "failed to check ephemeral-event community lifecycle"); + conn.send(RelayMessage::ok( + &event_id_hex, + false, + "error: internal server error", + )); + return; } - }; - if matches!(terminal, EphemeralTerminal::Accepted) { - if let Err(error) = serving_write.finish().await { - tracing::warn!(%error, event_id = %event_id_hex, "failed to release ephemeral-event serving lease"); + } + match handle_ephemeral_event( + event, + conn_id, + pubkey_bytes, + auth_pubkey, + Arc::clone(&conn), + state, + ) + .await + { + Ok(()) => { + conn.send(RelayMessage::ok(&event_id_hex, true, "")); + } + Err(message) => { + reject("invalid"); + conn.send(RelayMessage::ok(&event_id_hex, false, &message)); } } - conn.send(terminal.response(&event_id_hex)); return; } @@ -795,23 +791,6 @@ pub async fn handle_event(event: Event, conn: Arc, state: Arc String { - match self { - Self::Accepted => RelayMessage::ok(event_id, true, ""), - Self::Rejected(message) | Self::LeaseLost(message) => { - RelayMessage::ok(event_id, false, message) - } - } - } -} - /// Handle ephemeral events (kind 20000–29999) — WS-only, never stored. async fn handle_ephemeral_event( event: Event, @@ -1203,25 +1182,6 @@ mod tests { use tokio_util::sync::CancellationToken; use uuid::Uuid; - #[test] - fn ephemeral_terminal_builds_one_unambiguous_ok() { - let event_id = "abc"; - assert_eq!( - serde_json::from_str::( - &super::EphemeralTerminal::Accepted.response(event_id) - ) - .unwrap(), - serde_json::json!(["OK", event_id, true, ""]) - ); - assert_eq!( - serde_json::from_str::( - &super::EphemeralTerminal::LeaseLost("restricted".to_string()).response(event_id) - ) - .unwrap(), - serde_json::json!(["OK", event_id, false, "restricted"]) - ); - } - #[test] fn fanout_event_frame_matches_legacy_format_byte_for_byte() { let sub_id = "sub-id"; diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 21b71f378b..60737351c9 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -288,6 +288,33 @@ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ ]::TEXT[]) $$; +-- Fleet-wide writers must filter their candidate rows through this function +-- inside the mutating statement. The shared lock and lifecycle read form one +-- indivisible admission check, so a disallowed tenant is skipped before its +-- row trigger can abort healthy tenants in the same statement. +CREATE FUNCTION community_write_allowed(target UUID) RETURNS BOOLEAN +LANGUAGE plpgsql VOLATILE AS $$ +DECLARE + lifecycle TEXT; +BEGIN + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'community writes require READ COMMITTED isolation' + USING ERRCODE = 'invalid_transaction_state'; + END IF; + + IF target IS NULL THEN + RETURN true; + END IF; + + PERFORM pg_advisory_xact_lock_shared(community_deletion_lock_key(target)); + SELECT deletion_state + INTO lifecycle + FROM communities + WHERE id = target; + RETURN FOUND AND lifecycle = 'active'; +END +$$; + CREATE FUNCTION assert_community_write_allowed(target UUID) RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE @@ -302,6 +329,14 @@ DECLARE serving_fence_generation TEXT; serving_lease_valid BOOLEAN := false; BEGIN + -- The fence proof depends on a fresh statement snapshot after the shared + -- advisory lock is granted. Pinned RR/Serializable snapshots can retain a + -- pre-fence lifecycle or executor generation and resurrect tenant data. + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'community writes require READ COMMITTED isolation' + USING ERRCODE = 'invalid_transaction_state'; + END IF; + -- Nullable operator-attribution rows without a tenant are unrelated. IF target IS NULL THEN RETURN; diff --git a/schema/schema.sql b/schema/schema.sql index 53b30bb258..3e72f51e83 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1301,6 +1301,31 @@ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE AS $$ ]::TEXT[]) $$; +-- Fleet-wide writers filter candidates through this VOLATILE predicate in +-- the mutating statement so fenced tenants are skipped before row triggers run. +CREATE FUNCTION community_write_allowed(target UUID) RETURNS BOOLEAN +LANGUAGE plpgsql VOLATILE AS $$ +DECLARE + lifecycle TEXT; +BEGIN + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'community writes require READ COMMITTED isolation' + USING ERRCODE = 'invalid_transaction_state'; + END IF; + + IF target IS NULL THEN + RETURN true; + END IF; + + PERFORM pg_advisory_xact_lock_shared(community_deletion_lock_key(target)); + SELECT deletion_state + INTO lifecycle + FROM communities + WHERE id = target; + RETURN FOUND AND lifecycle = 'active'; +END +$$; + CREATE FUNCTION assert_community_write_allowed(target UUID) RETURNS VOID LANGUAGE plpgsql AS $$ DECLARE @@ -1315,6 +1340,13 @@ DECLARE serving_fence_generation TEXT; serving_lease_valid BOOLEAN := false; BEGIN + -- The fence proof requires a fresh statement snapshot after lock grant; + -- pinned RR/Serializable snapshots can retain pre-fence authorization. + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'community writes require READ COMMITTED isolation' + USING ERRCODE = 'invalid_transaction_state'; + END IF; + -- Nullable operator-attribution rows without a tenant are unrelated. IF target IS NULL THEN RETURN; diff --git a/scripts/check-community-fenced-writes.py b/scripts/check-community-fenced-writes.py new file mode 100755 index 0000000000..9e058fd2bc --- /dev/null +++ b/scripts/check-community-fenced-writes.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Structurally inventory production SQLx writes to community-fenced tables. + +ast-grep locates Rust call expressions, so multiline/raw strings and wrapper +formatting cannot disappear behind line-oriented grep. This classifier then +rejects fleet-wide UPDATE/DELETE statements unless the mutating statement +consumes community_write_allowed(community_id). Tenant-bound statements must +bind their community_id predicate directly. Dynamic SQL builders are tracked by +an explicit source inventory because their final SQL is assembled across calls. +""" + +from __future__ import annotations + +import argparse +import ast +import json +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +RULE = ROOT / "scripts/lints/community_fenced_writes.yml" +SOURCE_ROOTS = [ROOT / "crates/buzz-db/src", ROOT / "crates/buzz-audit/src", ROOT / "crates/buzz-relay/src"] +FENCED_TABLES = { + "api_tokens", "archived_identities", "audit_log", "channel_members", "channels", + "community_bans", "delivery_log", "event_mentions", "events", "git_repo_names", + "join_policy_acceptances", "moderation_actions", "moderation_reports", + "parameterized_event_watermarks", "product_feedback", "pubkey_allowlist", + "push_leases", "push_match_queue", "push_wake_outbox", "rate_limit_violations", + "reactions", "relay_invites", "relay_members", "scheduled_workflow_fires", + "subscriptions", "thread_metadata", "users", "workflow_approvals", "workflow_runs", + "workflows", +} +# Production dynamic write builders whose community binding was manually reviewed. +# The check fails if one moves or a new dynamic write builder appears. +DYNAMIC_WRITE_INVENTORY = { + ("crates/buzz-db/src/channel.rs", "UPDATE channels SET {}", "WHERE community_id = ${param_idx}"), + ("crates/buzz-db/src/deletion.rs", "DELETE FROM {table}", "WHERE community_id = $1"), + ("crates/buzz-db/src/lib.rs", "INSERT INTO event_mentions", "push_bind(community_id.as_uuid())"), + ("crates/buzz-db/src/user.rs", "UPDATE users SET {}", "WHERE community_id = ${param_idx}"), +} + +WRITE_RE = re.compile(r"\b(update|delete\s+from|insert\s+into|merge\s+into)\s+(?:only\s+)?(?:public\.)?\"?([a-z_][a-z0-9_]*)", re.I) +BOUND_RE = re.compile( + r"(?:\b[a-z_][a-z0-9_]*\.)?community_id\s*(?:=|is\s+not\s+distinct\s+from)\s*(?:any\s*\(\s*)?\$\d+" + r"|\$\d+\s*=\s*(?:\b[a-z_][a-z0-9_]*\.)?community_id", + re.I, +) + + +def rust_string_value(token: str) -> str | None: + raw = re.fullmatch(r'r(#{0,16})"(.*)"\1', token, re.S) + if raw: + return raw.group(2) + if token.startswith('"'): + try: + return ast.literal_eval(token) + except (SyntaxError, ValueError): + return None + return None + + +def production_prefix(path: Path) -> str: + text = path.read_text() + marker = text.find("#[cfg(test)]") + return text if marker < 0 else text[:marker] + + +def ast_matches() -> list[dict]: + command = [str(ROOT / "bin/ast-grep"), "scan", "--rule", str(RULE), "--json=stream", *map(str, SOURCE_ROOTS)] + try: + completed = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=True) + except FileNotFoundError: + raise SystemExit("bin/ast-grep is required; restore the repository Hermit package links") + except subprocess.CalledProcessError as error: + sys.stderr.write(error.stderr) + raise SystemExit(error.returncode) + return [json.loads(line) for line in completed.stdout.splitlines() if line.strip()] + + +def literal_from_match(match: dict) -> str | None: + sql = match.get("metaVariables", {}).get("single", {}).get("SQL", {}).get("text") + return rust_string_value(sql) if sql is not None else None + + +def classify(matches: list[dict]) -> tuple[list[str], set[tuple[str, int, str]]]: + violations: list[str] = [] + inventory: set[tuple[str, int, str]] = set() + for match in matches: + relative = str(Path(match["file"]).resolve().relative_to(ROOT)) + line = match["range"]["start"]["line"] + 1 + source_path = ROOT / relative + source = source_path.read_text() + # Exclude only the file's test module, not individual cfg(test) helpers + # that may occur inside the production impl before it. + test_module = re.search(r"(?m)^#\[cfg\(test\)\]\s*\n(?:pub\s+)?mod\s+tests\b", source) + if test_module and match["range"]["byteOffset"]["start"] > test_module.start(): + continue + sql = literal_from_match(match) + if sql is None: + continue + normalized = re.sub(r"\s+", " ", sql).strip().lower() + writes = [(op.lower(), table.lower()) for op, table in WRITE_RE.findall(normalized) if table.lower() in FENCED_TABLES] + if not writes: + continue + for operation, table in writes: + inventory.add((relative, line, table)) + if operation in {"update", "delete from", "merge into"}: + fleet = not BOUND_RE.search(normalized) + if fleet and "community_write_allowed(" not in normalized: + violations.append( + f"{relative}:{line}: fleet-wide {operation} of fenced table {table} " + "must consume community_write_allowed(community_id) in the writing statement" + ) + return violations, inventory + + +def dynamic_violations() -> list[str]: + violations: list[str] = [] + observed: set[tuple[str, str, str]] = set() + for relative, write_marker, scope_marker in DYNAMIC_WRITE_INVENTORY: + text = production_prefix(ROOT / relative) + if write_marker in text and scope_marker in text: + observed.add((relative, write_marker, scope_marker)) + else: + violations.append(f"{relative}: reviewed dynamic fenced-write contract changed: {write_marker!r} / {scope_marker!r}") + + dynamic_calls = [ + ("sqlx::query(AssertSqlSafe", re.compile(r"sqlx::(?:query|query_scalar|query_as)(?:<[^;]+?>)?\s*\(\s*(?:sqlx::)?AssertSqlSafe\s*\(")), + ("QueryBuilder::new", re.compile(r"QueryBuilder::new\s*\(")), + ] + for root in SOURCE_ROOTS: + for path in root.rglob("*.rs"): + text = production_prefix(path) + relative = str(path.relative_to(ROOT)) + for label, pattern in dynamic_calls: + for match in pattern.finditer(text): + # Inspect the local construction window, not the whole file; + # a read builder must not be confused with an unrelated write. + window = text[match.start() : match.start() + 900] + if not any( + re.search(rf"(?:UPDATE|DELETE FROM|INSERT INTO)\s+{table}\b", window, re.I) + for table in FENCED_TABLES + ): + continue + if not any( + entry[0] == relative + and entry in observed + and entry[1] in window + and entry[2] in text[max(0, match.start() - 500) : match.start() + 1800] + for entry in DYNAMIC_WRITE_INVENTORY + ): + line = text.count("\n", 0, match.start()) + 1 + violations.append(f"{relative}:{line}: unclassified dynamic SQL write builder ({label})") + return sorted(set(violations)) + + +def self_test(matches: list[dict]) -> None: + def violation(sql: str) -> bool: + normalized = re.sub(r"\s+", " ", sql).strip().lower() + writes = [(op.lower(), table.lower()) for op, table in WRITE_RE.findall(normalized) if table.lower() in FENCED_TABLES] + return any(op in {"update", "delete from", "merge into"} and not BOUND_RE.search(normalized) and "community_write_allowed(" not in normalized for op, _ in writes) + + assert violation("DELETE FROM relay_invites WHERE expires_at < $1") # known-bad positive control + assert violation("WITH c AS (SELECT * FROM events) UPDATE events SET d_tag='' WHERE d_tag IS NULL") + assert violation("DELETE FROM scheduled_workflow_fires WHERE claimed_at < $1") + assert not violation("DELETE FROM relay_invites WHERE community_id=$1 AND expires_at < $2") + assert not violation("DELETE FROM relay_invites WHERE expires_at < $1 AND community_write_allowed(community_id)") + assert not violation("SELECT * FROM relay_invites") + + # Calibrate the real structural extractor in both directions, not only the + # SQL classifier: the known production writers must be in its inventory. + expected_sites = { + ("crates/buzz-db/src/channel.rs", "channels"), + ("crates/buzz-db/src/lib.rs", "events"), + ("crates/buzz-db/src/push.rs", "push_match_queue"), + ("crates/buzz-db/src/relay_invite.rs", "relay_invites"), + ("crates/buzz-db/src/workflow.rs", "scheduled_workflow_fires"), + } + _, inventory = classify(matches) + observed_sites = {(relative, table) for relative, _, table in inventory} + missing = expected_sites - observed_sites + assert not missing, f"structural extractor missed calibrated production sites: {sorted(missing)}" + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--list", action="store_true", help="print the classified production write inventory") + args = parser.parse_args() + matches = ast_matches() + self_test(matches) + violations, inventory = classify(matches) + violations.extend(dynamic_violations()) + if args.list: + for relative, line, table in sorted(inventory): + print(f"{relative}:{line}: {table}") + if violations: + print("community fenced-writer structural lint failed:", file=sys.stderr) + for violation in violations: + print(f"- {violation}", file=sys.stderr) + return 1 + print(f"community fenced-writer structural lint passed ({len(inventory)} literal write sites classified)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/lints/community_fenced_writes.yml b/scripts/lints/community_fenced_writes.yml new file mode 100644 index 0000000000..39be303c09 --- /dev/null +++ b/scripts/lints/community_fenced_writes.yml @@ -0,0 +1,11 @@ +id: community-fenced-sqlx-write +message: SQLx call with a direct SQL expression; classified by check-community-fenced-writes.py +severity: hint +language: rust +rule: + any: + - pattern: sqlx::query($SQL) + - pattern: sqlx::query_scalar($SQL) + - pattern: sqlx::query_scalar::<$$$T>($SQL) + - pattern: sqlx::query_as($SQL) + - pattern: sqlx::query_as::<$$$T>($SQL) From 692fb59711bd868b9594cfb7507060438abf6680 Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 19:37:26 -0700 Subject: [PATCH 33/51] test: harden community writer inventory Co-authored-by: cid Signed-off-by: cid --- Cargo.lock | 13 + Justfile | 2 +- crates/buzz-db/Cargo.toml | 4 + .../buzz-db/tests/community_fenced_writes.rs | 573 ++++++++++++++++++ .../bad_insert_select.rs | 3 + .../bad_unrelated_subquery.rs | 3 + .../good_gated_delete.rs | 3 + .../good_tenant_delete.rs | 3 + scripts/check-community-fenced-writes.py | 208 ------- scripts/lints/community_fenced_writes.yml | 2 +- scripts/lints/community_test_modules.yml | 10 + 11 files changed, 614 insertions(+), 210 deletions(-) create mode 100644 crates/buzz-db/tests/community_fenced_writes.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_select.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_unrelated_subquery.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/good_gated_delete.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/good_tenant_delete.rs delete mode 100755 scripts/check-community-fenced-writes.py create mode 100644 scripts/lints/community_test_modules.yml diff --git a/Cargo.lock b/Cargo.lock index 6e064b2e5e..3309e83b28 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1006,18 +1006,22 @@ dependencies = [ "buzz-core", "chrono", "hex", + "litrs", "metrics", "metrics-util", "nostr", "rand 0.10.1", + "regex-lite", "serde", "serde_json", "sha2 0.11.0", + "sqlparser", "sqlx", "thiserror 2.0.18", "tokio", "tracing", "uuid", + "walkdir", ] [[package]] @@ -8958,6 +8962,15 @@ dependencies = [ "buzz-dev-mcp", ] +[[package]] +name = "sqlparser" +version = "0.62.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" +dependencies = [ + "log", +] + [[package]] name = "sqlx" version = "0.9.0" diff --git a/Justfile b/Justfile index 6aac156adf..8030d382dc 100644 --- a/Justfile +++ b/Justfile @@ -96,7 +96,7 @@ check: fmt-check community-fenced-writes-check clippy desktop-check desktop-taur # Structurally classify production SQL writes to community-fenced tables. community-fenced-writes-check: - ./scripts/check-community-fenced-writes.py + cargo test -p buzz-db --test community_fenced_writes # Format all Rust code fmt: diff --git a/crates/buzz-db/Cargo.toml b/crates/buzz-db/Cargo.toml index 6f76a11bc1..f1aa618d44 100644 --- a/crates/buzz-db/Cargo.toml +++ b/crates/buzz-db/Cargo.toml @@ -26,3 +26,7 @@ metrics = { workspace = true } [dev-dependencies] tokio = { workspace = true } metrics-util = { workspace = true } +sqlparser = { version = "0.62", default-features = false, features = ["std"] } +walkdir = "2" +litrs = "1" +regex-lite = "0.1" diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs new file mode 100644 index 0000000000..64a1bef96d --- /dev/null +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -0,0 +1,573 @@ +//! Repository-complete structural guard for production writes to community-fenced tables. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use sqlparser::ast::{ + BinaryOperator, Expr, FromTable, FunctionArg, FunctionArgExpr, FunctionArguments, Insert, + ObjectName, Query, Select, SetExpr, Statement, TableFactor, TableObject, TableWithJoins, Value, +}; +use sqlparser::dialect::PostgreSqlDialect; +use sqlparser::parser::Parser; + +const AUTHORITY: &[&str] = buzz_db::deletion::EXPECTED_SCOPED_TABLES; +const EXEMPT_CRATES: &[&str] = &[ + // Browser/CLI/protocol crates do not link SQLx or execute PostgreSQL. + "buzz-acp", + "buzz-agent", + "buzz-backend-kubernetes", + "buzz-cli", + "buzz-conformance", + "buzz-core", + "buzz-dev-mcp", + "buzz-media", + "buzz-pair-relay", + "buzz-pairing-cli", + "buzz-persona", + "buzz-pubsub", + "buzz-push-gateway", // deployment-global gateway tables only + "buzz-relay-mesh", + "buzz-sdk", + "buzz-test-client", // test-only client; production source has no SQL writers + "buzz-voice", + "buzz-workflow", + "buzz-ws-client", + "git-credential-nostr", + "git-sign-nostr", + "sprig", +]; +const NAMED_SQL_WRITES: &[(&str, &str, &str)] = &[( + "crates/buzz-db/src/reaction.rs", + "ADD_REACTION_SQL", + "ON CONFLICT (community_id, event_created_at, event_id, pubkey, emoji)", +)]; + +const CONDITIONAL_SQL_WRITES: &[(&str, &str, &[&str])] = &[( + "crates/buzz-db/src/lib.rs", + "let statement = if hard_delete_superseded", + &[ + "DELETE FROM events", + "UPDATE events SET deleted_at = NOW()", + "WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4", + ], +)]; + +const DYNAMIC_WRITES: &[(&str, &str, &str)] = &[ + ( + "crates/buzz-db/src/channel.rs", + "UPDATE channels SET {}", + "WHERE community_id = ${param_idx}", + ), + ( + "crates/buzz-db/src/deletion.rs", + "DELETE FROM {table}", + "WHERE community_id = $1", + ), + ( + "crates/buzz-db/src/lib.rs", + "INSERT INTO event_mentions", + "push_bind(community_id.as_uuid())", + ), + ( + "crates/buzz-db/src/user.rs", + "UPDATE users SET {}", + "WHERE community_id = ${param_idx}", + ), +]; + +// PostgreSQL syntax intentionally unsupported by sqlparser 0.62. Each entry is +// an exact, reviewed statement fingerprint; source edits fail closed until the +// statement either parses or this inventory is deliberately updated. +const PARSER_EXCEPTIONS: &[(&str, &str, &str)] = &[ + ( + "crates/buzz-db/src/deletion.rs", + "WITH candidate AS", + "WHERE request.id = candidate.id", + ), + ( + "crates/buzz-db/src/push.rs", + "WITH target AS", + "community_write_allowed(community_id)", + ), + ( + "crates/buzz-db/src/push.rs", + "WITH candidates AS", + "WHERE o.community_id = $1", + ), + ( + "crates/buzz-db/src/lib.rs", + "ON CONFLICT (lower(host))", + "communities.deletion_state = 'active'", + ), + ( + "crates/buzz-db/src/lib.rs", + "ON CONFLICT (lower(host)) DO NOTHING", + "RETURNING id, host", + ), +]; + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .and_then(Path::parent) + .expect("workspace root") + .to_owned() +} + +fn authority() -> BTreeSet { + AUTHORITY.iter().map(|table| (*table).to_owned()).collect() +} + +fn workspace_crates() -> BTreeSet { + fs::read_dir(repo_root().join("crates")) + .expect("read crates directory") + .filter_map(Result::ok) + .filter(|entry| entry.path().join("Cargo.toml").is_file()) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect() +} + +fn production_roots() -> Vec { + let exempt = EXEMPT_CRATES.iter().copied().collect::>(); + let crates = workspace_crates(); + let unknown_exempt = exempt + .iter() + .filter(|name| !crates.contains(**name)) + .copied() + .collect::>(); + assert!( + unknown_exempt.is_empty(), + "stale exempt crates: {unknown_exempt:?}" + ); + crates + .into_iter() + .filter(|name| !exempt.contains(name.as_str())) + .map(|name| repo_root().join("crates").join(name).join("src")) + .filter(|path| path.is_dir()) + .collect() +} + +fn ast_matches_for_rule(roots: &[PathBuf], rule: &str) -> Vec { + let mut command = Command::new(repo_root().join("bin/ast-grep")); + command + .arg("scan") + .arg("--rule") + .arg(repo_root().join(rule)) + .arg("--json=stream"); + command.args(roots); + let output = command.output().expect("run ast-grep"); + assert!( + output.status.success(), + "ast-grep failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout) + .expect("ast-grep utf8") + .lines() + .filter(|line| !line.is_empty()) + .map(|line| serde_json::from_str(line).expect("ast-grep json line")) + .collect() +} + +fn ast_matches(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule(roots, "scripts/lints/community_fenced_writes.yml") +} + +fn test_module_ranges(roots: &[PathBuf]) -> BTreeMap> { + let mut ranges = BTreeMap::>::new(); + for matched in ast_matches_for_rule(roots, "scripts/lints/community_test_modules.yml") { + let path = PathBuf::from(matched["file"].as_str().expect("test module file")); + let start = matched["range"]["byteOffset"]["start"] + .as_u64() + .expect("start offset"); + let end = matched["range"]["byteOffset"]["end"] + .as_u64() + .expect("end offset"); + ranges.entry(path).or_default().push((start, end)); + } + ranges +} + +fn rust_string_value(token: &str) -> Option { + if let Ok(literal) = litrs::StringLit::parse(token) { + return Some(literal.value().to_owned()); + } + if token.starts_with('"') && token.ends_with('"') { + let continued = regex_lite::Regex::new( + r"\\ +[ \t]*", + ) + .expect("line-continuation regex") + .replace_all(token, ""); + return serde_json::from_str(&continued).ok(); + } + None +} + +fn basename(name: &ObjectName) -> String { + name.0 + .last() + .and_then(|part| part.as_ident()) + .map(|ident| ident.value.to_ascii_lowercase()) + .unwrap_or_default() +} + +fn table_factor_name(factor: &TableFactor) -> Option { + match factor { + TableFactor::Table { name, .. } => Some(basename(name)), + _ => None, + } +} + +fn target_aliases(factor: &TableFactor, target: &str) -> BTreeSet { + let mut aliases = BTreeSet::from([target.to_owned()]); + if let TableFactor::Table { + alias: Some(alias), .. + } = factor + { + aliases.insert(alias.name.value.to_ascii_lowercase()); + } + aliases +} + +fn query_sources(query: &Query) -> BTreeSet { + fn collect_select(select: &Select, out: &mut BTreeSet) { + for TableWithJoins { relation, joins } in &select.from { + if let Some(name) = table_factor_name(relation) { + out.insert(name); + } + for join in joins { + if let Some(name) = table_factor_name(&join.relation) { + out.insert(name); + } + } + } + } + let mut out = BTreeSet::new(); + match query.body.as_ref() { + SetExpr::Select(select) => collect_select(select, &mut out), + SetExpr::Query(query) => out.extend(query_sources(query)), + SetExpr::SetOperation { left, right, .. } => { + if let SetExpr::Select(select) = left.as_ref() { + collect_select(select, &mut out); + } + if let SetExpr::Select(select) = right.as_ref() { + collect_select(select, &mut out); + } + } + _ => {} + } + out +} + +fn is_community_column(expr: &Expr, aliases: &BTreeSet) -> bool { + match expr { + Expr::Identifier(ident) => ident.value.eq_ignore_ascii_case("community_id"), + Expr::CompoundIdentifier(parts) if parts.len() >= 2 => { + parts + .last() + .is_some_and(|part| part.value.eq_ignore_ascii_case("community_id")) + && parts + .get(parts.len() - 2) + .is_some_and(|part| aliases.contains(&part.value.to_ascii_lowercase())) + } + _ => false, + } +} + +fn is_placeholder(expr: &Expr) -> bool { + matches!(expr, Expr::Value(value) if matches!(&value.value, Value::Placeholder(_))) +} + +fn is_allowed_function(expr: &Expr, aliases: &BTreeSet) -> bool { + let Expr::Function(function) = expr else { + return false; + }; + if basename(&function.name) != "community_write_allowed" { + return false; + } + let FunctionArguments::List(arguments) = &function.args else { + return false; + }; + matches!(arguments.args.as_slice(), [FunctionArg::Unnamed(FunctionArgExpr::Expr(argument))] if is_community_column(argument, aliases)) +} + +fn predicate_is_directly_gated(selection: Option<&Expr>, aliases: &BTreeSet) -> bool { + let Some(selection) = selection else { + return false; + }; + match selection { + Expr::Nested(expr) => predicate_is_directly_gated(Some(expr), aliases), + Expr::BinaryOp { + left, + op: BinaryOperator::And | BinaryOperator::Or, + right, + } => { + predicate_is_directly_gated(Some(left), aliases) + || predicate_is_directly_gated(Some(right), aliases) + } + Expr::BinaryOp { + left, + op: BinaryOperator::Eq, + right, + } => { + (is_community_column(left, aliases) && is_placeholder(right)) + || (is_placeholder(left) && is_community_column(right, aliases)) + } + Expr::InSubquery { subquery, .. } | Expr::Subquery(subquery) => { + query_is_tenant_bound_or_gated(subquery) + } + expr => is_allowed_function(expr, aliases), + } +} + +fn query_is_tenant_bound_or_gated(query: &Query) -> bool { + match query.body.as_ref() { + SetExpr::Select(select) => { + let aliases = select + .from + .iter() + .filter_map(|table| { + table_factor_name(&table.relation) + .map(|target| target_aliases(&table.relation, &target)) + }) + .flatten() + .collect::>(); + predicate_is_directly_gated(select.selection.as_ref(), &aliases) + } + SetExpr::Query(query) => query_is_tenant_bound_or_gated(query), + SetExpr::SetOperation { left, right, .. } => { + let branch = |set: &SetExpr| match set { + SetExpr::Select(select) => { + let aliases = select + .from + .iter() + .filter_map(|table| { + table_factor_name(&table.relation) + .map(|target| target_aliases(&table.relation, &target)) + }) + .flatten() + .collect::>(); + predicate_is_directly_gated(select.selection.as_ref(), &aliases) + } + SetExpr::Query(query) => query_is_tenant_bound_or_gated(query), + _ => false, + }; + branch(left) && branch(right) + } + SetExpr::Values(_) => true, + _ => false, + } +} + +fn mutation_violation(statement: &Statement, fenced: &BTreeSet) -> Option { + match statement { + Statement::Delete(delete) => { + let tables = match &delete.from { + FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables, + }; + let factor = &tables.first()?.relation; + let target = table_factor_name(factor)?; + if !fenced.contains(&target) { + return None; + } + let aliases = target_aliases(factor, &target); + (!predicate_is_directly_gated(delete.selection.as_ref(), &aliases)).then(|| { + format!("fleet DELETE from {target} lacks a direct tenant predicate or community_write_allowed") + }) + } + Statement::Update(update) => { + let factor = &update.table.relation; + let target = table_factor_name(factor)?; + if !fenced.contains(&target) { + return None; + } + let aliases = target_aliases(factor, &target); + (!predicate_is_directly_gated(update.selection.as_ref(), &aliases)).then(|| { + format!("fleet UPDATE of {target} lacks a direct tenant predicate or community_write_allowed") + }) + } + Statement::Insert(Insert { table, source, .. }) => { + let TableObject::TableName(table_name) = table else { + return None; + }; + let target = basename(table_name); + if !fenced.contains(&target) { + return None; + } + let Some(source) = source else { return None }; + let sources = query_sources(source); + if sources.iter().any(|source| fenced.contains(source)) + && !query_is_tenant_bound_or_gated(source) + { + return Some(format!( + "fleet INSERT SELECT into {target} reads fenced sources without a direct gate" + )); + } + None + } + _ => None, + } +} + +fn parse_sql(sql: &str) -> Result, String> { + Parser::parse_sql(&PostgreSqlDialect {}, sql).map_err(|error| error.to_string()) +} + +fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec { + let fenced = authority(); + let root = repo_root(); + let mut violations = Vec::new(); + let mut observed_parser_exceptions = BTreeSet::new(); + let test_ranges = test_module_ranges(roots); + let production_files = roots + .iter() + .flat_map(|source| { + walkdir::WalkDir::new(source) + .into_iter() + .filter_map(Result::ok) + }) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "rs")) + .map(|entry| entry.into_path()) + .collect::>(); + assert!(!production_files.is_empty()); + + for matched in matches { + let path = PathBuf::from(matched["file"].as_str().expect("match file")); + assert!( + production_files.contains(&path), + "match escaped production roots: {path:?}" + ); + let offset = matched["range"]["byteOffset"]["start"] + .as_u64() + .expect("match offset"); + if test_ranges.get(&path).is_some_and(|ranges| { + ranges + .iter() + .any(|(start, end)| *start <= offset && offset < *end) + }) { + continue; + } + let line = matched["range"]["start"]["line"].as_u64().unwrap_or(0) + 1; + let sql_token = matched["metaVariables"]["single"]["SQL"]["text"] + .as_str() + .expect("SQL metavariable"); + let Some(sql) = rust_string_value(sql_token) else { + let relative = path.strip_prefix(&root).unwrap_or(&path).to_string_lossy(); + let inventoried = NAMED_SQL_WRITES.iter().any(|(file, symbol, guard)| { + relative == *file + && sql_token.contains(symbol) + && fs::read_to_string(&path) + .is_ok_and(|source| source.contains(symbol) && source.contains(guard)) + }); + let conditional = CONDITIONAL_SQL_WRITES.iter().any(|(file, symbol, guards)| { + relative == *file + && sql_token == "statement" + && fs::read_to_string(&path).is_ok_and(|source| { + source.contains(symbol) && guards.iter().all(|guard| source.contains(guard)) + }) + }); + if !inventoried && !conditional && !sql_token.contains("AssertSqlSafe") { + violations.push(format!( + "{relative}:{line}: unclassified non-literal SQL expression: {sql_token}" + )); + } + continue; + }; + match parse_sql(&sql) { + Ok(statements) => { + for statement in statements { + if let Some(message) = mutation_violation(&statement, &fenced) { + violations.push(format!( + "{}:{line}: {message}", + path.strip_prefix(&root).unwrap_or(&path).display() + )); + } + } + } + Err(error) + if sql.to_ascii_lowercase().contains("insert into") + || sql.to_ascii_lowercase().contains("update ") + || sql.to_ascii_lowercase().contains("delete from") => + { + let relative = path.strip_prefix(&root).unwrap_or(&path).to_string_lossy(); + let excepted = PARSER_EXCEPTIONS.iter().any(|(file, start, guard)| { + relative == *file && sql.contains(start) && sql.contains(guard) + }); + if excepted { + observed_parser_exceptions.insert(relative.into_owned()); + } else { + violations.push(format!( + "{relative}:{line}: production write SQL did not parse: {error}" + )); + } + } + Err(_) => {} + } + } + + for (relative, write_marker, scope_marker) in DYNAMIC_WRITES { + let text = fs::read_to_string(root.join(relative)).expect("read dynamic writer"); + if !text.contains(write_marker) || !text.contains(scope_marker) { + violations.push(format!( + "{relative}: dynamic writer contract changed: {write_marker:?} / {scope_marker:?}" + )); + } + } + for (relative, symbol, guards) in CONDITIONAL_SQL_WRITES { + let text = fs::read_to_string(root.join(relative)).expect("read conditional SQL writer"); + if !text.contains(symbol) || !guards.iter().all(|guard| text.contains(guard)) { + violations.push(format!( + "{relative}: conditional SQL writer contract changed: {symbol:?} / {guards:?}" + )); + } + } + for (relative, symbol, guard) in NAMED_SQL_WRITES { + let text = fs::read_to_string(root.join(relative)).expect("read named SQL writer"); + if !text.contains(symbol) || !text.contains(guard) { + violations.push(format!( + "{relative}: named SQL writer contract changed: {symbol:?} / {guard:?}" + )); + } + } + violations +} + +#[test] +fn production_fenced_writers_are_structurally_tenant_bound_or_gated() { + let roots = production_roots(); + let matches = ast_matches(&roots); + let violations = inspect_matches(&matches, &roots); + assert!( + violations.is_empty(), + "community fenced-writer violations:\n{}", + violations.join("\n") + ); +} + +#[test] +fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { + let fixture = repo_root().join("crates/buzz-db/tests/fixtures/community_fenced_writes"); + let matches = ast_matches(std::slice::from_ref(&fixture)); + let violations = inspect_matches(&matches, &[fixture]); + assert!( + violations.iter().any(|v| v.contains("bad_insert_select")), + "fleet INSERT SELECT control escaped: {violations:?}" + ); + assert!( + violations + .iter() + .any(|v| v.contains("bad_unrelated_subquery")), + "unrelated-subquery DELETE control escaped: {violations:?}" + ); + assert!( + !violations.iter().any(|v| v.contains("good_tenant_delete")), + "tenant-bound control was rejected: {violations:?}" + ); + assert!( + !violations.iter().any(|v| v.contains("good_gated_delete")), + "gated control was rejected: {violations:?}" + ); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_select.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_select.rs new file mode 100644 index 0000000000..e46a8931ae --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_select.rs @@ -0,0 +1,3 @@ +fn bad_insert_select() { + sqlx::query("INSERT INTO relay_invites (community_id, token_hash, expires_at, created_by) SELECT community_id, token_hash, now(), 'x' FROM relay_invites"); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_unrelated_subquery.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_unrelated_subquery.rs new file mode 100644 index 0000000000..ad7b80f2c7 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_unrelated_subquery.rs @@ -0,0 +1,3 @@ +fn bad_unrelated_subquery() { + sqlx::query("DELETE FROM relay_invites WHERE expires_at < now() AND EXISTS (SELECT 1 FROM communities WHERE id=$1)"); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_gated_delete.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_gated_delete.rs new file mode 100644 index 0000000000..5ef73d84b0 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_gated_delete.rs @@ -0,0 +1,3 @@ +fn good_gated_delete() { + sqlx::query("DELETE FROM relay_invites WHERE expires_at < now() AND community_write_allowed(community_id)"); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_tenant_delete.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_tenant_delete.rs new file mode 100644 index 0000000000..2d578d5bd0 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_tenant_delete.rs @@ -0,0 +1,3 @@ +fn good_tenant_delete() { + sqlx::query("DELETE FROM relay_invites WHERE community_id=$1 AND expires_at < $2"); +} diff --git a/scripts/check-community-fenced-writes.py b/scripts/check-community-fenced-writes.py deleted file mode 100755 index 9e058fd2bc..0000000000 --- a/scripts/check-community-fenced-writes.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 -"""Structurally inventory production SQLx writes to community-fenced tables. - -ast-grep locates Rust call expressions, so multiline/raw strings and wrapper -formatting cannot disappear behind line-oriented grep. This classifier then -rejects fleet-wide UPDATE/DELETE statements unless the mutating statement -consumes community_write_allowed(community_id). Tenant-bound statements must -bind their community_id predicate directly. Dynamic SQL builders are tracked by -an explicit source inventory because their final SQL is assembled across calls. -""" - -from __future__ import annotations - -import argparse -import ast -import json -import re -import subprocess -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -RULE = ROOT / "scripts/lints/community_fenced_writes.yml" -SOURCE_ROOTS = [ROOT / "crates/buzz-db/src", ROOT / "crates/buzz-audit/src", ROOT / "crates/buzz-relay/src"] -FENCED_TABLES = { - "api_tokens", "archived_identities", "audit_log", "channel_members", "channels", - "community_bans", "delivery_log", "event_mentions", "events", "git_repo_names", - "join_policy_acceptances", "moderation_actions", "moderation_reports", - "parameterized_event_watermarks", "product_feedback", "pubkey_allowlist", - "push_leases", "push_match_queue", "push_wake_outbox", "rate_limit_violations", - "reactions", "relay_invites", "relay_members", "scheduled_workflow_fires", - "subscriptions", "thread_metadata", "users", "workflow_approvals", "workflow_runs", - "workflows", -} -# Production dynamic write builders whose community binding was manually reviewed. -# The check fails if one moves or a new dynamic write builder appears. -DYNAMIC_WRITE_INVENTORY = { - ("crates/buzz-db/src/channel.rs", "UPDATE channels SET {}", "WHERE community_id = ${param_idx}"), - ("crates/buzz-db/src/deletion.rs", "DELETE FROM {table}", "WHERE community_id = $1"), - ("crates/buzz-db/src/lib.rs", "INSERT INTO event_mentions", "push_bind(community_id.as_uuid())"), - ("crates/buzz-db/src/user.rs", "UPDATE users SET {}", "WHERE community_id = ${param_idx}"), -} - -WRITE_RE = re.compile(r"\b(update|delete\s+from|insert\s+into|merge\s+into)\s+(?:only\s+)?(?:public\.)?\"?([a-z_][a-z0-9_]*)", re.I) -BOUND_RE = re.compile( - r"(?:\b[a-z_][a-z0-9_]*\.)?community_id\s*(?:=|is\s+not\s+distinct\s+from)\s*(?:any\s*\(\s*)?\$\d+" - r"|\$\d+\s*=\s*(?:\b[a-z_][a-z0-9_]*\.)?community_id", - re.I, -) - - -def rust_string_value(token: str) -> str | None: - raw = re.fullmatch(r'r(#{0,16})"(.*)"\1', token, re.S) - if raw: - return raw.group(2) - if token.startswith('"'): - try: - return ast.literal_eval(token) - except (SyntaxError, ValueError): - return None - return None - - -def production_prefix(path: Path) -> str: - text = path.read_text() - marker = text.find("#[cfg(test)]") - return text if marker < 0 else text[:marker] - - -def ast_matches() -> list[dict]: - command = [str(ROOT / "bin/ast-grep"), "scan", "--rule", str(RULE), "--json=stream", *map(str, SOURCE_ROOTS)] - try: - completed = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, check=True) - except FileNotFoundError: - raise SystemExit("bin/ast-grep is required; restore the repository Hermit package links") - except subprocess.CalledProcessError as error: - sys.stderr.write(error.stderr) - raise SystemExit(error.returncode) - return [json.loads(line) for line in completed.stdout.splitlines() if line.strip()] - - -def literal_from_match(match: dict) -> str | None: - sql = match.get("metaVariables", {}).get("single", {}).get("SQL", {}).get("text") - return rust_string_value(sql) if sql is not None else None - - -def classify(matches: list[dict]) -> tuple[list[str], set[tuple[str, int, str]]]: - violations: list[str] = [] - inventory: set[tuple[str, int, str]] = set() - for match in matches: - relative = str(Path(match["file"]).resolve().relative_to(ROOT)) - line = match["range"]["start"]["line"] + 1 - source_path = ROOT / relative - source = source_path.read_text() - # Exclude only the file's test module, not individual cfg(test) helpers - # that may occur inside the production impl before it. - test_module = re.search(r"(?m)^#\[cfg\(test\)\]\s*\n(?:pub\s+)?mod\s+tests\b", source) - if test_module and match["range"]["byteOffset"]["start"] > test_module.start(): - continue - sql = literal_from_match(match) - if sql is None: - continue - normalized = re.sub(r"\s+", " ", sql).strip().lower() - writes = [(op.lower(), table.lower()) for op, table in WRITE_RE.findall(normalized) if table.lower() in FENCED_TABLES] - if not writes: - continue - for operation, table in writes: - inventory.add((relative, line, table)) - if operation in {"update", "delete from", "merge into"}: - fleet = not BOUND_RE.search(normalized) - if fleet and "community_write_allowed(" not in normalized: - violations.append( - f"{relative}:{line}: fleet-wide {operation} of fenced table {table} " - "must consume community_write_allowed(community_id) in the writing statement" - ) - return violations, inventory - - -def dynamic_violations() -> list[str]: - violations: list[str] = [] - observed: set[tuple[str, str, str]] = set() - for relative, write_marker, scope_marker in DYNAMIC_WRITE_INVENTORY: - text = production_prefix(ROOT / relative) - if write_marker in text and scope_marker in text: - observed.add((relative, write_marker, scope_marker)) - else: - violations.append(f"{relative}: reviewed dynamic fenced-write contract changed: {write_marker!r} / {scope_marker!r}") - - dynamic_calls = [ - ("sqlx::query(AssertSqlSafe", re.compile(r"sqlx::(?:query|query_scalar|query_as)(?:<[^;]+?>)?\s*\(\s*(?:sqlx::)?AssertSqlSafe\s*\(")), - ("QueryBuilder::new", re.compile(r"QueryBuilder::new\s*\(")), - ] - for root in SOURCE_ROOTS: - for path in root.rglob("*.rs"): - text = production_prefix(path) - relative = str(path.relative_to(ROOT)) - for label, pattern in dynamic_calls: - for match in pattern.finditer(text): - # Inspect the local construction window, not the whole file; - # a read builder must not be confused with an unrelated write. - window = text[match.start() : match.start() + 900] - if not any( - re.search(rf"(?:UPDATE|DELETE FROM|INSERT INTO)\s+{table}\b", window, re.I) - for table in FENCED_TABLES - ): - continue - if not any( - entry[0] == relative - and entry in observed - and entry[1] in window - and entry[2] in text[max(0, match.start() - 500) : match.start() + 1800] - for entry in DYNAMIC_WRITE_INVENTORY - ): - line = text.count("\n", 0, match.start()) + 1 - violations.append(f"{relative}:{line}: unclassified dynamic SQL write builder ({label})") - return sorted(set(violations)) - - -def self_test(matches: list[dict]) -> None: - def violation(sql: str) -> bool: - normalized = re.sub(r"\s+", " ", sql).strip().lower() - writes = [(op.lower(), table.lower()) for op, table in WRITE_RE.findall(normalized) if table.lower() in FENCED_TABLES] - return any(op in {"update", "delete from", "merge into"} and not BOUND_RE.search(normalized) and "community_write_allowed(" not in normalized for op, _ in writes) - - assert violation("DELETE FROM relay_invites WHERE expires_at < $1") # known-bad positive control - assert violation("WITH c AS (SELECT * FROM events) UPDATE events SET d_tag='' WHERE d_tag IS NULL") - assert violation("DELETE FROM scheduled_workflow_fires WHERE claimed_at < $1") - assert not violation("DELETE FROM relay_invites WHERE community_id=$1 AND expires_at < $2") - assert not violation("DELETE FROM relay_invites WHERE expires_at < $1 AND community_write_allowed(community_id)") - assert not violation("SELECT * FROM relay_invites") - - # Calibrate the real structural extractor in both directions, not only the - # SQL classifier: the known production writers must be in its inventory. - expected_sites = { - ("crates/buzz-db/src/channel.rs", "channels"), - ("crates/buzz-db/src/lib.rs", "events"), - ("crates/buzz-db/src/push.rs", "push_match_queue"), - ("crates/buzz-db/src/relay_invite.rs", "relay_invites"), - ("crates/buzz-db/src/workflow.rs", "scheduled_workflow_fires"), - } - _, inventory = classify(matches) - observed_sites = {(relative, table) for relative, _, table in inventory} - missing = expected_sites - observed_sites - assert not missing, f"structural extractor missed calibrated production sites: {sorted(missing)}" - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--list", action="store_true", help="print the classified production write inventory") - args = parser.parse_args() - matches = ast_matches() - self_test(matches) - violations, inventory = classify(matches) - violations.extend(dynamic_violations()) - if args.list: - for relative, line, table in sorted(inventory): - print(f"{relative}:{line}: {table}") - if violations: - print("community fenced-writer structural lint failed:", file=sys.stderr) - for violation in violations: - print(f"- {violation}", file=sys.stderr) - return 1 - print(f"community fenced-writer structural lint passed ({len(inventory)} literal write sites classified)") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/lints/community_fenced_writes.yml b/scripts/lints/community_fenced_writes.yml index 39be303c09..23ad682bcb 100644 --- a/scripts/lints/community_fenced_writes.yml +++ b/scripts/lints/community_fenced_writes.yml @@ -1,5 +1,5 @@ id: community-fenced-sqlx-write -message: SQLx call with a direct SQL expression; classified by check-community-fenced-writes.py +message: SQLx call classified by the buzz-db community_fenced_writes integration test severity: hint language: rust rule: diff --git a/scripts/lints/community_test_modules.yml b/scripts/lints/community_test_modules.yml new file mode 100644 index 0000000000..e269a82f5a --- /dev/null +++ b/scripts/lints/community_test_modules.yml @@ -0,0 +1,10 @@ +id: community-test-module +message: Rust test module excluded from production SQL writer inventory +severity: hint +language: rust +rule: + kind: mod_item + has: + field: name + regex: '^tests$' + stopBy: end From 035daa3c84775ff94793819877ffe820217a359a Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 20:01:35 -0700 Subject: [PATCH 34/51] fix: compose writer connection guards Co-authored-by: cid Signed-off-by: cid --- crates/buzz-db/src/lib.rs | 117 +++++- .../buzz-db/tests/community_fenced_writes.rs | 376 ++++++++++++------ .../bad_dynamic_assert_sql_safe.rs | 3 + .../bad_insert_unfenced_source.rs | 3 + .../bad_insert_unrelated_alias.rs | 3 + .../community_fenced_writes/bad_or_delete.rs | 3 + scripts/lints/community_sqlx_calls.yml | 11 + 7 files changed, 374 insertions(+), 142 deletions(-) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_assert_sql_safe.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unfenced_source.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unrelated_alias.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_or_delete.rs create mode 100644 scripts/lints/community_sqlx_calls.yml diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 3e1864fb7c..e9ace6fd90 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -651,7 +651,7 @@ impl Db { /// `buzz.created_at_floor` GUC — this is what makes the replica fence /// proof hold for every insert path that goes through this pool. pub async fn new(config: &DbConfig) -> Result { - let pool = Self::connect_pool(config, &config.database_url, true).await?; + let pool = Self::connect_pool(config, &config.database_url).await?; let read_max_connections = config .read_max_connections .unwrap_or(config.max_connections); @@ -671,31 +671,39 @@ impl Db { }) } - /// Connect one pool with the sizing knobs from `config`. + /// Connect the writer pool with all session-level safety premises. /// - /// `arm_floor_guard` sets the `buzz.created_at_floor` session GUC on - /// every connection, arming the deferred commit-time trigger from - /// migration 0021. Writer pools must arm it; replica pools are read-only - /// so the trigger never fires there. - async fn connect_pool(config: &DbConfig, url: &str, arm_floor_guard: bool) -> Result { - let mut options = PgPoolOptions::new() + /// SQLx stores one `after_connect` hook, so the floor guard and transaction + /// isolation assertion must remain in this single closure. Registering a + /// second hook replaces the first and silently disarms the floor trigger. + async fn connect_pool(config: &DbConfig, url: &str) -> Result { + let options = PgPoolOptions::new() .max_connections(config.max_connections) .min_connections(config.min_connections) .acquire_timeout(Duration::from_secs(config.acquire_timeout_secs)) .max_lifetime(Duration::from_secs(config.max_lifetime_secs)) - .idle_timeout(Duration::from_secs(config.idle_timeout_secs)); - if arm_floor_guard { - options = options.after_connect(|conn, _meta| { + .idle_timeout(Duration::from_secs(config.idle_timeout_secs)) + .after_connect(|conn, _meta| { Box::pin(async move { // `SET` cannot take bind parameters; `set_config` can. sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(conn) + .execute(&mut *conn) + .await?; + let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") + .fetch_one(&mut *conn) .await?; + if isolation != "read committed" { + return Err(sqlx::Error::Configuration( + format!( + "writer pool requires READ COMMITTED transaction isolation, got {isolation}" + ) + .into(), + )); + } Ok(()) }) }); - } Ok(options.connect(url).await?) } @@ -720,8 +728,9 @@ impl Db { /// are dialed only on first acquire; the ~10-minute reaper never tops /// the pool back up, which is fine — routed reads re-fill it on demand. /// - /// No floor guard: replica sessions are read-only, the trigger never - /// fires there (see [`Db::connect_pool`]). + /// No floor guard or writer-isolation assertion: replica sessions are + /// read-only, so the commit-time trigger from migration 0021 never fires + /// here and the write fence that depends on READ COMMITTED is never reached. fn connect_read_pool(config: &DbConfig, url: &str, max_connections: u32) -> Result { Ok(PgPoolOptions::new() .max_connections(max_connections) @@ -8374,6 +8383,76 @@ mod tests { drop_scratch_db(&admin, pool, &name).await; } + #[test] + fn writer_pool_safety_hook_is_single_and_composed() { + let source = include_str!("lib.rs"); + let connect_pool = source + .split("async fn connect_pool") + .nth(1) + .and_then(|tail| tail.split("const READER_ACQUIRE_TIMEOUT").next()) + .expect("connect_pool source block"); + assert_eq!( + connect_pool.matches(".after_connect(").count(), + 1, + "SQLx replaces after_connect hooks; writer safety must use exactly one" + ); + assert!(connect_pool.contains("buzz.created_at_floor")); + assert!(connect_pool.contains("SHOW transaction_isolation")); + assert!(!connect_pool.contains("arm_floor_guard")); + assert!(!connect_pool.contains("_arm_floor_guard")); + assert!(!connect_pool.contains("allow(unused_variables)")); + + let reader_doc = source + .split("fn connect_read_pool") + .next() + .and_then(|prefix| prefix.rsplit("/// Connect the read-replica").next()) + .expect("reader pool documentation"); + assert!(reader_doc.contains("replica sessions are")); + assert!(reader_doc.contains("read-only")); + assert!(!reader_doc.contains("Db::connect_pool")); + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn writer_pool_rejects_non_read_committed_database_default() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (seed_pool, name) = create_scratch_db(&admin, "writer_isolation").await; + sqlx::query(sqlx::AssertSqlSafe(format!( + "ALTER DATABASE {name} SET default_transaction_isolation = 'repeatable read'" + ))) + .execute(&admin) + .await + .expect("set unsafe database default"); + seed_pool.close().await; + + let base = admin_url().await; + let idx = base.rfind('/').expect("db url has a path segment"); + let scratch_url = format!("{}/{}", &base[..idx], name); + let error = Db::new(&DbConfig { + database_url: scratch_url, + max_connections: 1, + min_connections: 1, + acquire_timeout_secs: 1, + ..DbConfig::default() + }) + .await + .expect_err("writer pool must reject pinned-snapshot database defaults"); + assert!( + error.to_string().contains("requires READ COMMITTED") + || error.to_string().contains("pool timed out"), + "unexpected isolation rejection: {error}" + ); + + sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE {name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop isolation test database"); + } + /// The armed writer pool (`Db::new`) must enforce the floor end-to-end /// through the public insert APIs, and the session GUC must be verifiably /// set on pooled connections. @@ -8413,6 +8492,14 @@ mod tests { crate::replica_fence::CREATED_AT_FLOOR_SECS.to_string(), "writer pool must arm the floor guard on every connection" ); + let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") + .fetch_one(&db.pool) + .await + .expect("SHOW writer isolation"); + assert_eq!( + isolation, "read committed", + "the same writer after_connect hook must enforce the isolation premise" + ); let now_secs = chrono::Utc::now().timestamp() as u64; let floor = crate::replica_fence::CREATED_AT_FLOOR_SECS as u64; diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index 64a1bef96d..ce594655ae 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -7,37 +7,12 @@ use std::process::Command; use sqlparser::ast::{ BinaryOperator, Expr, FromTable, FunctionArg, FunctionArgExpr, FunctionArguments, Insert, - ObjectName, Query, Select, SetExpr, Statement, TableFactor, TableObject, TableWithJoins, Value, + ObjectName, Query, Select, SelectItem, SetExpr, Statement, TableFactor, TableObject, Value, }; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; const AUTHORITY: &[&str] = buzz_db::deletion::EXPECTED_SCOPED_TABLES; -const EXEMPT_CRATES: &[&str] = &[ - // Browser/CLI/protocol crates do not link SQLx or execute PostgreSQL. - "buzz-acp", - "buzz-agent", - "buzz-backend-kubernetes", - "buzz-cli", - "buzz-conformance", - "buzz-core", - "buzz-dev-mcp", - "buzz-media", - "buzz-pair-relay", - "buzz-pairing-cli", - "buzz-persona", - "buzz-pubsub", - "buzz-push-gateway", // deployment-global gateway tables only - "buzz-relay-mesh", - "buzz-sdk", - "buzz-test-client", // test-only client; production source has no SQL writers - "buzz-voice", - "buzz-workflow", - "buzz-ws-client", - "git-credential-nostr", - "git-sign-nostr", - "sprig", -]; const NAMED_SQL_WRITES: &[(&str, &str, &str)] = &[( "crates/buzz-db/src/reaction.rs", "ADD_REACTION_SQL", @@ -54,26 +29,87 @@ const CONDITIONAL_SQL_WRITES: &[(&str, &str, &[&str])] = &[( ], )]; -const DYNAMIC_WRITES: &[(&str, &str, &str)] = &[ +const DYNAMIC_SQL: &[(&str, &str, &[&str])] = &[ + ( + "crates/buzz-db/src/channel.rs", + "get_accessible_channels", + &["FROM channels c", "WHERE c.community_id = $1"], + ), ( "crates/buzz-db/src/channel.rs", - "UPDATE channels SET {}", - "WHERE community_id = ${param_idx}", + "get_users_bulk", + &["FROM users WHERE community_id = $1"], + ), + ( + "crates/buzz-db/src/channel.rs", + "update_channel", + &[ + "UPDATE channels SET {}", + "WHERE community_id = ${param_idx}", + ], ), ( "crates/buzz-db/src/deletion.rs", - "DELETE FROM {table}", - "WHERE community_id = $1", + "inventory_schema", + &["SELECT count(*)::BIGINT FROM {table} WHERE community_id = $1"], + ), + ( + "crates/buzz-db/src/deletion.rs", + "purge_postgres", + &["DELETE FROM {table} WHERE community_id = $1"], + ), + ( + "crates/buzz-db/src/deletion.rs", + "verify_postgres_logically_deleted", + &["SELECT EXISTS(SELECT 1 FROM {table} WHERE community_id = $1 LIMIT 1)"], ), ( "crates/buzz-db/src/lib.rs", - "INSERT INTO event_mentions", - "push_bind(community_id.as_uuid())", + "insert_mentions", + &[ + "INSERT INTO event_mentions", + "push_bind(community_id.as_uuid())", + ], + ), + ( + "crates/buzz-db/src/partition.rs", + "ensure_partition", + &["CREATE TABLE IF NOT EXISTS {partition_name} PARTITION OF {table_name}"], + ), + ( + "crates/buzz-db/src/replica_fence.rs", + "reader_supports_aurora_identity", + &["SELECT {AURORA_IDENTITY_FN}()"], + ), + ( + "crates/buzz-db/src/replica_fence.rs", + "observe_heartbeat", + &["FROM replica_heartbeat WHERE id = 1"], + ), + ( + "crates/buzz-db/src/thread.rs", + "get_thread_replies_on", + &["WHERE tm.community_id = $1"], + ), + ( + "crates/buzz-db/src/thread.rs", + "get_channel_window_on", + &["WHERE e.community_id = $1"], + ), + ( + "crates/buzz-db/src/usage.rs", + "active_user_counts", + &["SELECT", "FROM events e"], + ), + ( + "crates/buzz-db/src/usage.rs", + "active_channel_counts", + &["SELECT community_id", "FROM events"], ), ( "crates/buzz-db/src/user.rs", - "UPDATE users SET {}", - "WHERE community_id = ${param_idx}", + "update_user_profile", + &["UPDATE users SET {}", "WHERE community_id = ${param_idx}"], ), ]; @@ -130,23 +166,13 @@ fn workspace_crates() -> BTreeSet { } fn production_roots() -> Vec { - let exempt = EXEMPT_CRATES.iter().copied().collect::>(); - let crates = workspace_crates(); - let unknown_exempt = exempt - .iter() - .filter(|name| !crates.contains(**name)) - .copied() - .collect::>(); - assert!( - unknown_exempt.is_empty(), - "stale exempt crates: {unknown_exempt:?}" - ); - crates + let roots = workspace_crates() .into_iter() - .filter(|name| !exempt.contains(name.as_str())) .map(|name| repo_root().join("crates").join(name).join("src")) .filter(|path| path.is_dir()) - .collect() + .collect::>(); + assert!(!roots.is_empty(), "workspace has no production crate roots"); + roots } fn ast_matches_for_rule(roots: &[PathBuf], rule: &str) -> Vec { @@ -175,6 +201,10 @@ fn ast_matches(roots: &[PathBuf]) -> Vec { ast_matches_for_rule(roots, "scripts/lints/community_fenced_writes.yml") } +fn all_sqlx_matches(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule(roots, "scripts/lints/community_sqlx_calls.yml") +} + fn test_module_ranges(roots: &[PathBuf]) -> BTreeMap> { let mut ranges = BTreeMap::>::new(); for matched in ast_matches_for_rule(roots, "scripts/lints/community_test_modules.yml") { @@ -232,36 +262,6 @@ fn target_aliases(factor: &TableFactor, target: &str) -> BTreeSet { aliases } -fn query_sources(query: &Query) -> BTreeSet { - fn collect_select(select: &Select, out: &mut BTreeSet) { - for TableWithJoins { relation, joins } in &select.from { - if let Some(name) = table_factor_name(relation) { - out.insert(name); - } - for join in joins { - if let Some(name) = table_factor_name(&join.relation) { - out.insert(name); - } - } - } - } - let mut out = BTreeSet::new(); - match query.body.as_ref() { - SetExpr::Select(select) => collect_select(select, &mut out), - SetExpr::Query(query) => out.extend(query_sources(query)), - SetExpr::SetOperation { left, right, .. } => { - if let SetExpr::Select(select) = left.as_ref() { - collect_select(select, &mut out); - } - if let SetExpr::Select(select) = right.as_ref() { - collect_select(select, &mut out); - } - } - _ => {} - } - out -} - fn is_community_column(expr: &Expr, aliases: &BTreeSet) -> bool { match expr { Expr::Identifier(ident) => ident.value.eq_ignore_ascii_case("community_id"), @@ -302,12 +302,20 @@ fn predicate_is_directly_gated(selection: Option<&Expr>, aliases: &BTreeSet predicate_is_directly_gated(Some(expr), aliases), Expr::BinaryOp { left, - op: BinaryOperator::And | BinaryOperator::Or, + op: BinaryOperator::And, right, } => { predicate_is_directly_gated(Some(left), aliases) || predicate_is_directly_gated(Some(right), aliases) } + Expr::BinaryOp { + left, + op: BinaryOperator::Or, + right, + } => { + predicate_is_directly_gated(Some(left), aliases) + && predicate_is_directly_gated(Some(right), aliases) + } Expr::BinaryOp { left, op: BinaryOperator::Eq, @@ -317,51 +325,102 @@ fn predicate_is_directly_gated(selection: Option<&Expr>, aliases: &BTreeSet { - query_is_tenant_bound_or_gated(subquery) + query_contains_direct_gate(subquery) } expr => is_allowed_function(expr, aliases), } } -fn query_is_tenant_bound_or_gated(query: &Query) -> bool { +fn select_aliases(select: &Select) -> BTreeSet { + select + .from + .iter() + .filter_map(|table| { + table_factor_name(&table.relation) + .map(|target| target_aliases(&table.relation, &target)) + }) + .flatten() + .collect() +} + +fn select_projection_expr(select: &Select, index: usize) -> Option<&Expr> { + match select.projection.get(index)? { + SelectItem::UnnamedExpr(expr) + | SelectItem::ExprWithAlias { expr, .. } + | SelectItem::ExprWithAliases { expr, .. } => Some(expr), + SelectItem::QualifiedWildcard(..) | SelectItem::Wildcard(..) => None, + } +} + +fn query_contains_direct_gate(query: &Query) -> bool { match query.body.as_ref() { SetExpr::Select(select) => { - let aliases = select - .from - .iter() - .filter_map(|table| { - table_factor_name(&table.relation) - .map(|target| target_aliases(&table.relation, &target)) - }) - .flatten() - .collect::>(); - predicate_is_directly_gated(select.selection.as_ref(), &aliases) + predicate_is_directly_gated(select.selection.as_ref(), &select_aliases(select)) } - SetExpr::Query(query) => query_is_tenant_bound_or_gated(query), + SetExpr::Query(query) => query_contains_direct_gate(query), SetExpr::SetOperation { left, right, .. } => { - let branch = |set: &SetExpr| match set { - SetExpr::Select(select) => { - let aliases = select - .from - .iter() - .filter_map(|table| { - table_factor_name(&table.relation) - .map(|target| target_aliases(&table.relation, &target)) - }) - .flatten() - .collect::>(); - predicate_is_directly_gated(select.selection.as_ref(), &aliases) + set_expr_contains_direct_gate(left) && set_expr_contains_direct_gate(right) + } + _ => false, + } +} + +fn set_expr_contains_direct_gate(set: &SetExpr) -> bool { + match set { + SetExpr::Select(select) => { + predicate_is_directly_gated(select.selection.as_ref(), &select_aliases(select)) + } + SetExpr::Query(query) => query_contains_direct_gate(query), + SetExpr::SetOperation { left, right, .. } => { + set_expr_contains_direct_gate(left) && set_expr_contains_direct_gate(right) + } + _ => false, + } +} + +fn select_projected_community_is_safe(select: &Select, index: usize) -> bool { + let aliases = select_aliases(select); + let Some(projected) = select_projection_expr(select, index) else { + return false; + }; + match projected { + Expr::Value(value) if matches!(&value.value, Value::Placeholder(_)) => true, + expr if is_allowed_function(expr, &aliases) => true, + expr if is_community_column(expr, &aliases) => { + let projected_alias = match expr { + Expr::CompoundIdentifier(parts) if parts.len() >= 2 => { + BTreeSet::from([parts[parts.len() - 2].value.to_ascii_lowercase()]) } - SetExpr::Query(query) => query_is_tenant_bound_or_gated(query), - _ => false, + Expr::Identifier(_) if select.from.len() == 1 => aliases, + _ => return false, }; - branch(left) && branch(right) + predicate_is_directly_gated(select.selection.as_ref(), &projected_alias) } - SetExpr::Values(_) => true, _ => false, } } +fn set_projected_community_is_safe(set: &SetExpr, index: usize) -> bool { + match set { + SetExpr::Select(select) => select_projected_community_is_safe(select, index), + SetExpr::Query(query) => projected_community_is_safe(query, index), + SetExpr::SetOperation { left, right, .. } => { + set_projected_community_is_safe(left, index) + && set_projected_community_is_safe(right, index) + } + SetExpr::Values(values) => values.rows.iter().all(|row| { + row.get(index).is_some_and(|expr| { + matches!(expr, Expr::Value(value) if matches!(&value.value, Value::Placeholder(_))) + }) + }), + _ => false, + } +} + +fn projected_community_is_safe(query: &Query, index: usize) -> bool { + set_projected_community_is_safe(query.body.as_ref(), index) +} + fn mutation_violation(statement: &Statement, fenced: &BTreeSet) -> Option { match statement { Statement::Delete(delete) => { @@ -389,7 +448,12 @@ fn mutation_violation(statement: &Statement, fenced: &BTreeSet) -> Optio format!("fleet UPDATE of {target} lacks a direct tenant predicate or community_write_allowed") }) } - Statement::Insert(Insert { table, source, .. }) => { + Statement::Insert(Insert { + table, + columns, + source, + .. + }) => { let TableObject::TableName(table_name) = table else { return None; }; @@ -397,16 +461,15 @@ fn mutation_violation(statement: &Statement, fenced: &BTreeSet) -> Optio if !fenced.contains(&target) { return None; } + let community_index = columns + .iter() + .position(|column| basename(column) == "community_id")?; let Some(source) = source else { return None }; - let sources = query_sources(source); - if sources.iter().any(|source| fenced.contains(source)) - && !query_is_tenant_bound_or_gated(source) - { - return Some(format!( - "fleet INSERT SELECT into {target} reads fenced sources without a direct gate" - )); - } - None + (!projected_community_is_safe(source, community_index)).then(|| { + format!( + "INSERT into {target} does not prove target community_id is tenant-bound or gated" + ) + }) } _ => None, } @@ -421,7 +484,14 @@ fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec Vec Vec Vec Vec>(); + let matches = all_sqlx_matches(&roots); + assert!(!matches.is_empty(), "SQLx coverage probe matched no calls"); + for matched in matches { + let path = PathBuf::from(matched["file"].as_str().expect("SQLx call file")); + assert!( + production_files.contains(&path), + "SQLx call escaped workspace source roots: {path:?}" + ); + } +} + #[test] fn production_fenced_writers_are_structurally_tenant_bound_or_gated() { let roots = production_roots(); @@ -562,6 +662,28 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { .any(|v| v.contains("bad_unrelated_subquery")), "unrelated-subquery DELETE control escaped: {violations:?}" ); + for (fixture_name, description) in [ + ("bad_or_delete", "OR-broadened DELETE"), + ( + "bad_insert_unfenced_source", + "unfenced-source INSERT SELECT", + ), + ( + "bad_insert_unrelated_alias", + "unrelated-alias INSERT SELECT", + ), + ( + "bad_dynamic_assert_sql_safe", + "unreviewed AssertSqlSafe dynamic SQL", + ), + ] { + assert!( + violations + .iter() + .any(|violation| violation.contains(fixture_name)), + "{description} control escaped: {violations:?}" + ); + } assert!( !violations.iter().any(|v| v.contains("good_tenant_delete")), "tenant-bound control was rejected: {violations:?}" diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_assert_sql_safe.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_assert_sql_safe.rs new file mode 100644 index 0000000000..16b27bce29 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_assert_sql_safe.rs @@ -0,0 +1,3 @@ +fn bad_dynamic_assert_sql_safe(dynamic_sql: String) { + sqlx::query(sqlx::AssertSqlSafe(dynamic_sql)); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unfenced_source.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unfenced_source.rs new file mode 100644 index 0000000000..917fdc5002 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unfenced_source.rs @@ -0,0 +1,3 @@ +fn bad_insert_unfenced_source() { + sqlx::query("INSERT INTO relay_invites (community_id, token_hash, expires_at, created_by) SELECT id, decode(repeat('00', 32), 'hex'), now(), 'x' FROM communities"); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unrelated_alias.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unrelated_alias.rs new file mode 100644 index 0000000000..ba6b35e9b4 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unrelated_alias.rs @@ -0,0 +1,3 @@ +fn bad_insert_unrelated_alias() { + sqlx::query("INSERT INTO relay_invites (community_id, token_hash, expires_at, created_by) SELECT ri.community_id, ri.token_hash, now(), 'x' FROM relay_invites ri, event_tags et WHERE et.community_id = $1"); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_or_delete.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_or_delete.rs new file mode 100644 index 0000000000..b80442d764 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_or_delete.rs @@ -0,0 +1,3 @@ +fn bad_or_delete() { + sqlx::query("DELETE FROM relay_invites WHERE community_id = $1 OR expires_at < now()"); +} diff --git a/scripts/lints/community_sqlx_calls.yml b/scripts/lints/community_sqlx_calls.yml new file mode 100644 index 0000000000..8684a79407 --- /dev/null +++ b/scripts/lints/community_sqlx_calls.yml @@ -0,0 +1,11 @@ +id: community-sqlx-call +message: Any SQLx execution call used to prove repository-complete source coverage +severity: hint +language: rust +rule: + any: + - pattern: sqlx::query($SQL) + - pattern: sqlx::query_scalar($SQL) + - pattern: sqlx::query_scalar::<$$$T>($SQL) + - pattern: sqlx::query_as($SQL) + - pattern: sqlx::query_as::<$$$T>($SQL) From bb2cab462e21b7e8d3565f133d777bbdb9585c19 Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 20:09:24 -0700 Subject: [PATCH 35/51] test: close writer scanner escapes Co-authored-by: cid Signed-off-by: cid --- .../buzz-db/tests/community_fenced_writes.rs | 132 ++++++++++++++---- .../bad_insert_implicit_columns.rs | 3 + .../bad_production_mod_tests.rs | 5 + .../good_cfg_test_module.rs | 6 + .../lints/community_cfg_test_attributes.yml | 7 + scripts/lints/community_function_items.yml | 6 + scripts/lints/community_modules.yml | 6 + scripts/lints/community_test_modules.yml | 10 -- 8 files changed, 134 insertions(+), 41 deletions(-) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_implicit_columns.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_production_mod_tests.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/good_cfg_test_module.rs create mode 100644 scripts/lints/community_cfg_test_attributes.yml create mode 100644 scripts/lints/community_function_items.yml create mode 100644 scripts/lints/community_modules.yml delete mode 100644 scripts/lints/community_test_modules.yml diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index ce594655ae..b177ca70f5 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -206,20 +206,68 @@ fn all_sqlx_matches(roots: &[PathBuf]) -> Vec { } fn test_module_ranges(roots: &[PathBuf]) -> BTreeMap> { + let mut cfg_offsets = BTreeMap::>::new(); + for matched in ast_matches_for_rule(roots, "scripts/lints/community_cfg_test_attributes.yml") { + let path = PathBuf::from(matched["file"].as_str().expect("cfg(test) file")); + let offset = matched["range"]["byteOffset"]["start"] + .as_u64() + .expect("cfg offset"); + cfg_offsets.entry(path).or_default().push(offset); + } + let mut ranges = BTreeMap::>::new(); - for matched in ast_matches_for_rule(roots, "scripts/lints/community_test_modules.yml") { - let path = PathBuf::from(matched["file"].as_str().expect("test module file")); + for matched in ast_matches_for_rule(roots, "scripts/lints/community_modules.yml") { + let path = PathBuf::from(matched["file"].as_str().expect("module file")); let start = matched["range"]["byteOffset"]["start"] .as_u64() - .expect("start offset"); + .expect("module start"); let end = matched["range"]["byteOffset"]["end"] .as_u64() - .expect("end offset"); - ranges.entry(path).or_default().push((start, end)); + .expect("module end"); + let source = fs::read_to_string(&path).expect("read module source"); + let attributed = cfg_offsets.get(&path).is_some_and(|offsets| { + offsets.iter().any(|offset| { + *offset < start + && source[*offset as usize..start as usize] + .trim() + .starts_with("#[cfg(test)]") + }) + }); + if attributed { + ranges.entry(path).or_default().push((start, end)); + } } ranges } +#[derive(Debug)] +struct FunctionOwner { + path: PathBuf, + start: u64, + end: u64, + name: String, + text: String, +} + +fn function_owners(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule(roots, "scripts/lints/community_function_items.yml") + .into_iter() + .filter_map(|matched| { + let text = matched["text"].as_str()?.to_owned(); + let signature = regex_lite::Regex::new(r"(?:async\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)") + .expect("function signature regex"); + let name = signature.captures(&text)?.get(1)?.as_str().to_owned(); + Some(FunctionOwner { + path: PathBuf::from(matched["file"].as_str()?), + start: matched["range"]["byteOffset"]["start"].as_u64()?, + end: matched["range"]["byteOffset"]["end"].as_u64()?, + name, + text, + }) + }) + .collect() +} + fn rust_string_value(token: &str) -> Option { if let Ok(literal) = litrs::StringLit::parse(token) { return Some(literal.value().to_owned()); @@ -484,14 +532,8 @@ fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec Vec Vec Date: Wed, 5 Aug 2026 20:17:12 -0700 Subject: [PATCH 36/51] test: inventory sql execution APIs Co-authored-by: cid Signed-off-by: cid --- .../buzz-db/tests/community_fenced_writes.rs | 73 +++++++++++++++++-- .../bad_generic_query.rs | 3 + .../community_fenced_writes/bad_query_with.rs | 3 + .../community_fenced_writes/bad_raw_sql.rs | 3 + scripts/lints/community_fenced_writes.yml | 7 +- .../lints/community_querybuilder_builds.yml | 9 +++ .../community_querybuilder_creations.yml | 10 +++ scripts/lints/community_sqlx_calls.yml | 7 +- 8 files changed, 104 insertions(+), 11 deletions(-) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_generic_query.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_with.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_raw_sql.rs create mode 100644 scripts/lints/community_querybuilder_builds.yml create mode 100644 scripts/lints/community_querybuilder_creations.yml diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index b177ca70f5..31a4cd06f0 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -30,6 +30,14 @@ const CONDITIONAL_SQL_WRITES: &[(&str, &str, &[&str])] = &[( )]; const DYNAMIC_SQL: &[(&str, &str, &[&str])] = &[ + ( + "crates/buzz-push-gateway/src/postgres.rs", + "apply_migrations_and_grants", + &[ + "REVOKE CREATE ON DATABASE {database}", + "GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE", + ], + ), ( "crates/buzz-db/src/channel.rs", "get_accessible_channels", @@ -205,6 +213,14 @@ fn all_sqlx_matches(roots: &[PathBuf]) -> Vec { ast_matches_for_rule(roots, "scripts/lints/community_sqlx_calls.yml") } +fn querybuilder_creations(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule(roots, "scripts/lints/community_querybuilder_creations.yml") +} + +fn querybuilder_builds(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule(roots, "scripts/lints/community_querybuilder_builds.yml") +} + fn test_module_ranges(roots: &[PathBuf]) -> BTreeMap> { let mut cfg_offsets = BTreeMap::>::new(); for matched in ast_matches_for_rule(roots, "scripts/lints/community_cfg_test_attributes.yml") { @@ -563,9 +579,15 @@ fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec Vec>(); let matches = all_sqlx_matches(&roots); - assert!(!matches.is_empty(), "SQLx coverage probe matched no calls"); + assert!( + !matches.is_empty(), + "SQL execution coverage probe matched no calls" + ); + let owners = function_owners(&roots); + let creations = querybuilder_creations(&roots); for matched in matches { - let path = PathBuf::from(matched["file"].as_str().expect("SQLx call file")); + let path = PathBuf::from(matched["file"].as_str().expect("SQL execution file")); assert!( production_files.contains(&path), - "SQLx call escaped workspace source roots: {path:?}" + "SQL execution escaped workspace roots: {path:?}" + ); + } + for build in querybuilder_builds(&roots) { + let path = PathBuf::from(build["file"].as_str().expect("builder file")); + let offset = build["range"]["byteOffset"]["start"] + .as_u64() + .expect("builder offset"); + let owner = owners + .iter() + .find(|owner| owner.path == path && owner.start <= offset && offset < owner.end); + let Some(owner) = owner else { continue }; + if !owner.text.contains("QueryBuilder") { + continue; + } + let receiver = build["metaVariables"]["single"]["QB"]["text"] + .as_str() + .expect("builder receiver"); + let owned_creations = creations + .iter() + .filter(|creation| { + path == Path::new(creation["file"].as_str().unwrap_or_default()) + && creation["range"]["byteOffset"]["start"] + .as_u64() + .is_some_and(|start| owner.start <= start && start < owner.end) + && (owner.text.contains(&format!("{receiver}:")) + || owner.text.contains(&format!("let mut {receiver} ="))) + }) + .count(); + assert!( + owned_creations >= 1, + "QueryBuilder execution in {} has no exact owned constructor for receiver {receiver}", + path.display() ); } } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_generic_query.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_generic_query.rs new file mode 100644 index 0000000000..ce4ab38735 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_generic_query.rs @@ -0,0 +1,3 @@ +fn mutant(dynamic_sql: String) { + sqlx::query::(sqlx::AssertSqlSafe(dynamic_sql)); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_with.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_with.rs new file mode 100644 index 0000000000..b0abb5a5f8 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_with.rs @@ -0,0 +1,3 @@ +fn mutant(dynamic_sql: String, args: sqlx::postgres::PgArguments) { + sqlx::query_with(sqlx::AssertSqlSafe(dynamic_sql), args); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_raw_sql.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_raw_sql.rs new file mode 100644 index 0000000000..5e866cdb81 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_raw_sql.rs @@ -0,0 +1,3 @@ +fn mutant(dynamic_sql: String) { + sqlx::raw_sql(sqlx::AssertSqlSafe(dynamic_sql)); +} diff --git a/scripts/lints/community_fenced_writes.yml b/scripts/lints/community_fenced_writes.yml index 23ad682bcb..32467b1958 100644 --- a/scripts/lints/community_fenced_writes.yml +++ b/scripts/lints/community_fenced_writes.yml @@ -1,11 +1,14 @@ -id: community-fenced-sqlx-write -message: SQLx call classified by the buzz-db community_fenced_writes integration test +id: community-sql-execution +message: SQL execution constructor/builder used to prove repository-complete source coverage severity: hint language: rust rule: any: - pattern: sqlx::query($SQL) + - pattern: sqlx::query::<$$$T>($SQL) - pattern: sqlx::query_scalar($SQL) - pattern: sqlx::query_scalar::<$$$T>($SQL) - pattern: sqlx::query_as($SQL) - pattern: sqlx::query_as::<$$$T>($SQL) + - pattern: sqlx::query_with($SQL, $ARGS) + - pattern: sqlx::raw_sql($SQL) diff --git a/scripts/lints/community_querybuilder_builds.yml b/scripts/lints/community_querybuilder_builds.yml new file mode 100644 index 0000000000..6187da1d44 --- /dev/null +++ b/scripts/lints/community_querybuilder_builds.yml @@ -0,0 +1,9 @@ +id: community-querybuilder-build +message: QueryBuilder execution call bound to its exact receiver symbol +severity: hint +language: rust +rule: + any: + - pattern: $QB.build() + - pattern: $QB.build_query_as::<$$$T>() + - pattern: $QB.build_query_scalar::<$$$T>() diff --git a/scripts/lints/community_querybuilder_creations.yml b/scripts/lints/community_querybuilder_creations.yml new file mode 100644 index 0000000000..020ee5f6da --- /dev/null +++ b/scripts/lints/community_querybuilder_creations.yml @@ -0,0 +1,10 @@ +id: community-querybuilder-creation +message: QueryBuilder construction bound to its owning function +severity: hint +language: rust +rule: + any: + - pattern: QueryBuilder::new($SQL) + - pattern: QueryBuilder::<$$$T>::new($SQL) + - pattern: sqlx::QueryBuilder::new($SQL) + - pattern: sqlx::QueryBuilder::<$$$T>::new($SQL) diff --git a/scripts/lints/community_sqlx_calls.yml b/scripts/lints/community_sqlx_calls.yml index 8684a79407..32467b1958 100644 --- a/scripts/lints/community_sqlx_calls.yml +++ b/scripts/lints/community_sqlx_calls.yml @@ -1,11 +1,14 @@ -id: community-sqlx-call -message: Any SQLx execution call used to prove repository-complete source coverage +id: community-sql-execution +message: SQL execution constructor/builder used to prove repository-complete source coverage severity: hint language: rust rule: any: - pattern: sqlx::query($SQL) + - pattern: sqlx::query::<$$$T>($SQL) - pattern: sqlx::query_scalar($SQL) - pattern: sqlx::query_scalar::<$$$T>($SQL) - pattern: sqlx::query_as($SQL) - pattern: sqlx::query_as::<$$$T>($SQL) + - pattern: sqlx::query_with($SQL, $ARGS) + - pattern: sqlx::raw_sql($SQL) From 5a074009e19258a4b0a19e2d5deb1d82390a7c2e Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 20:23:32 -0700 Subject: [PATCH 37/51] test: classify query builder writes Co-authored-by: cid Signed-off-by: cid --- .../buzz-db/tests/community_fenced_writes.rs | 154 +++++++++++++----- .../bad_query_as_with.rs | 1 + .../bad_query_scalar_with.rs | 1 + .../bad_querybuilder_direct.rs | 1 + .../bad_querybuilder_split.rs | 1 + scripts/lints/community_fenced_writes.yml | 8 +- .../community_querybuilder_mutations.yml | 11 ++ scripts/lints/community_sqlx_calls.yml | 8 +- 8 files changed, 136 insertions(+), 49 deletions(-) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_as_with.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_scalar_with.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_direct.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_split.rs create mode 100644 scripts/lints/community_querybuilder_mutations.yml diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index 31a4cd06f0..b3b2615423 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -213,8 +213,8 @@ fn all_sqlx_matches(roots: &[PathBuf]) -> Vec { ast_matches_for_rule(roots, "scripts/lints/community_sqlx_calls.yml") } -fn querybuilder_creations(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule(roots, "scripts/lints/community_querybuilder_creations.yml") +fn querybuilder_mutations(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule(roots, "scripts/lints/community_querybuilder_mutations.yml") } fn querybuilder_builds(roots: &[PathBuf]) -> Vec { @@ -525,9 +525,19 @@ fn mutation_violation(statement: &Statement, fenced: &BTreeSet) -> Optio if !fenced.contains(&target) { return None; } - let community_index = columns + if columns.is_empty() { + return Some(format!( + "INSERT into {target} omits its target column list; community_id provenance is unknown" + )); + } + let Some(community_index) = columns .iter() - .position(|column| basename(column) == "community_id")?; + .position(|column| basename(column) == "community_id") + else { + return Some(format!( + "INSERT into {target} omits community_id from its target columns" + )); + }; let Some(source) = source else { return None }; (!projected_community_is_safe(source, community_index)).then(|| { format!( @@ -697,34 +707,11 @@ fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec>(); - let matches = all_sqlx_matches(&roots); - assert!( - !matches.is_empty(), - "SQL execution coverage probe matched no calls" - ); - let owners = function_owners(&roots); - let creations = querybuilder_creations(&roots); - for matched in matches { - let path = PathBuf::from(matched["file"].as_str().expect("SQL execution file")); - assert!( - production_files.contains(&path), - "SQL execution escaped workspace roots: {path:?}" - ); - } - for build in querybuilder_builds(&roots) { +fn querybuilder_violations(roots: &[PathBuf]) -> Vec { + let owners = function_owners(roots); + let mutations = querybuilder_mutations(roots); + let mut found = Vec::new(); + for build in querybuilder_builds(roots) { let path = PathBuf::from(build["file"].as_str().expect("builder file")); let offset = build["range"]["byteOffset"]["start"] .as_u64() @@ -739,23 +726,86 @@ fn every_workspace_sql_execution_is_scanned_and_querybuilders_are_owned() { let receiver = build["metaVariables"]["single"]["QB"]["text"] .as_str() .expect("builder receiver"); - let owned_creations = creations + let fragments = mutations .iter() - .filter(|creation| { - path == Path::new(creation["file"].as_str().unwrap_or_default()) - && creation["range"]["byteOffset"]["start"] + .filter(|mutation| { + path == Path::new(mutation["file"].as_str().unwrap_or_default()) + && mutation["range"]["byteOffset"]["start"] .as_u64() .is_some_and(|start| owner.start <= start && start < owner.end) - && (owner.text.contains(&format!("{receiver}:")) - || owner.text.contains(&format!("let mut {receiver} ="))) }) - .count(); + .filter_map(|mutation| mutation["metaVariables"]["single"].get("SQL")?["text"].as_str()) + .filter_map(rust_string_value) + .collect::>(); + if fragments.is_empty() { + found.push(format!( + "{}: QueryBuilder {receiver} has no literal fragments", + path.display() + )); + continue; + } + let reconstructed = fragments.join(" "); + if !["insert into", "update ", "delete from"] + .iter() + .any(|keyword| reconstructed.to_ascii_lowercase().contains(keyword)) + { + continue; + } + let relative = path + .strip_prefix(repo_root()) + .unwrap_or(&path) + .to_string_lossy(); + let statements = match parse_sql(&reconstructed) { + Ok(statements) => statements, + Err(error) => { + let inventoried = DYNAMIC_SQL.iter().any(|(file, expected_owner, guards)| { + relative == *file + && owner.name == *expected_owner + && guards.iter().all(|guard| owner.text.contains(guard)) + }); + if !inventoried { + found.push(format!("{}: mutating QueryBuilder {receiver} failed reconstruction and has no exact owner inventory: {error}; fragments={fragments:?}", path.display())); + } + continue; + } + }; + let fenced = authority(); + found.extend(statements.iter().filter_map(|statement| { + mutation_violation(statement, &fenced).map(|violation| { + format!( + "{}: mutating QueryBuilder {receiver}: {violation}", + path.display() + ) + }) + })); + } + found +} + +#[test] +fn every_workspace_sql_execution_is_scanned_and_querybuilders_are_owned() { + let roots = production_roots(); + let matches = all_sqlx_matches(&roots); + assert!( + !matches.is_empty(), + "SQL execution coverage probe matched no calls" + ); + for matched in &matches { assert!( - owned_creations >= 1, - "QueryBuilder execution in {} has no exact owned constructor for receiver {receiver}", - path.display() + matched["file"].as_str().is_some(), + "SQL execution match has no source file" ); } + let violations = querybuilder_violations(&roots); + assert!( + violations.is_empty(), + "QueryBuilder violations: +{}", + violations.join( + " +" + ) + ); } #[test] @@ -774,7 +824,7 @@ fn production_fenced_writers_are_structurally_tenant_bound_or_gated() { fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { let fixture = repo_root().join("crates/buzz-db/tests/fixtures/community_fenced_writes"); let matches = ast_matches(std::slice::from_ref(&fixture)); - let violations = inspect_matches(&matches, &[fixture]); + let violations = inspect_matches(&matches, std::slice::from_ref(&fixture)); assert!( violations.iter().any(|v| v.contains("bad_insert_select")), "fleet INSERT SELECT control escaped: {violations:?}" @@ -799,6 +849,15 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { "bad_dynamic_assert_sql_safe", "unreviewed AssertSqlSafe dynamic SQL", ), + ( + "bad_insert_implicit_columns", + "implicit-column INSERT SELECT", + ), + ("bad_generic_query", "generic query constructor"), + ("bad_query_with", "query_with constructor"), + ("bad_query_as_with", "query_as_with constructor"), + ("bad_query_scalar_with", "query_scalar_with constructor"), + ("bad_raw_sql", "raw_sql constructor"), ] { assert!( violations @@ -807,6 +866,15 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { "{description} control escaped: {violations:?}" ); } + let builder_violations = querybuilder_violations(std::slice::from_ref(&fixture)); + for fixture_name in ["bad_querybuilder_direct", "bad_querybuilder_split"] { + assert!( + builder_violations + .iter() + .any(|violation| violation.contains(fixture_name)), + "QueryBuilder control {fixture_name} escaped: {builder_violations:?}" + ); + } assert!( !violations.iter().any(|v| v.contains("good_tenant_delete")), "tenant-bound control was rejected: {violations:?}" diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_as_with.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_as_with.rs new file mode 100644 index 0000000000..8d6c4af26d --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_as_with.rs @@ -0,0 +1 @@ +fn mutant(sql: String, args: sqlx::postgres::PgArguments) { sqlx::query_as_with::(sqlx::AssertSqlSafe(sql), args); } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_scalar_with.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_scalar_with.rs new file mode 100644 index 0000000000..9d02c49135 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_scalar_with.rs @@ -0,0 +1 @@ +fn mutant(sql: String, args: sqlx::postgres::PgArguments) { sqlx::query_scalar_with::(sqlx::AssertSqlSafe(sql), args); } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_direct.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_direct.rs new file mode 100644 index 0000000000..4d257aaac4 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_direct.rs @@ -0,0 +1 @@ +fn mutant() { let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at < now()"); let _ = qb.build(); } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_split.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_split.rs new file mode 100644 index 0000000000..bdb3e195bb --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_split.rs @@ -0,0 +1 @@ +fn mutant() { let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE "); qb.push("expires_at < now()"); let _ = qb.build(); } diff --git a/scripts/lints/community_fenced_writes.yml b/scripts/lints/community_fenced_writes.yml index 32467b1958..bc3223d12a 100644 --- a/scripts/lints/community_fenced_writes.yml +++ b/scripts/lints/community_fenced_writes.yml @@ -6,9 +6,11 @@ rule: any: - pattern: sqlx::query($SQL) - pattern: sqlx::query::<$$$T>($SQL) - - pattern: sqlx::query_scalar($SQL) - - pattern: sqlx::query_scalar::<$$$T>($SQL) + - pattern: sqlx::query_with($SQL, $ARGS) - pattern: sqlx::query_as($SQL) - pattern: sqlx::query_as::<$$$T>($SQL) - - pattern: sqlx::query_with($SQL, $ARGS) + - pattern: sqlx::query_as_with::<$$$T>($SQL, $ARGS) + - pattern: sqlx::query_scalar($SQL) + - pattern: sqlx::query_scalar::<$$$T>($SQL) + - pattern: sqlx::query_scalar_with::<$$$T>($SQL, $ARGS) - pattern: sqlx::raw_sql($SQL) diff --git a/scripts/lints/community_querybuilder_mutations.yml b/scripts/lints/community_querybuilder_mutations.yml new file mode 100644 index 0000000000..5ee66db043 --- /dev/null +++ b/scripts/lints/community_querybuilder_mutations.yml @@ -0,0 +1,11 @@ +id: community-querybuilder-mutation +message: QueryBuilder constructor/push fragments used to reconstruct mutating builders +severity: hint +language: rust +rule: + any: + - pattern: QueryBuilder::new($SQL) + - pattern: QueryBuilder::<$$$T>::new($SQL) + - pattern: sqlx::QueryBuilder::new($SQL) + - pattern: sqlx::QueryBuilder::<$$$T>::new($SQL) + - pattern: $QB.push($SQL) diff --git a/scripts/lints/community_sqlx_calls.yml b/scripts/lints/community_sqlx_calls.yml index 32467b1958..bc3223d12a 100644 --- a/scripts/lints/community_sqlx_calls.yml +++ b/scripts/lints/community_sqlx_calls.yml @@ -6,9 +6,11 @@ rule: any: - pattern: sqlx::query($SQL) - pattern: sqlx::query::<$$$T>($SQL) - - pattern: sqlx::query_scalar($SQL) - - pattern: sqlx::query_scalar::<$$$T>($SQL) + - pattern: sqlx::query_with($SQL, $ARGS) - pattern: sqlx::query_as($SQL) - pattern: sqlx::query_as::<$$$T>($SQL) - - pattern: sqlx::query_with($SQL, $ARGS) + - pattern: sqlx::query_as_with::<$$$T>($SQL, $ARGS) + - pattern: sqlx::query_scalar($SQL) + - pattern: sqlx::query_scalar::<$$$T>($SQL) + - pattern: sqlx::query_scalar_with::<$$$T>($SQL, $ARGS) - pattern: sqlx::raw_sql($SQL) From ff4ce731190e53400f09a1ffdab029e9bae7d9df Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 20:29:15 -0700 Subject: [PATCH 38/51] test: calibrate writer pool hooks Co-authored-by: cid Signed-off-by: cid --- crates/buzz-db/src/lib.rs | 139 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 139 insertions(+) diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index e9ace6fd90..6f3cc3461b 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -60,6 +60,8 @@ pub use error::{DbError, Result}; pub use event::{EventQuery, ReactionEventInsertOutcome, DEFAULT_MAX_PAGE_LIMIT}; use chrono::{DateTime, Utc}; +#[cfg(test)] +use sqlx::postgres::PgConnectOptions; use sqlx::postgres::{PgConnection, PgPoolOptions}; use sqlx::{Connection, PgPool, QueryBuilder, Row}; use std::time::Duration; @@ -500,6 +502,67 @@ impl UsageMetricsLeader { } } +#[cfg(test)] +#[derive(Clone, Copy)] +enum WriterHookMode { + Combined, + Separate, + CombinedWithoutIsolation, + None, +} + +#[cfg(test)] +fn writer_pool_options_for_mode(mode: WriterHookMode) -> PgPoolOptions { + let floor = |options: PgPoolOptions| { + options.after_connect(|conn, _meta| { + Box::pin(async move { + sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") + .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) + .execute(conn) + .await?; + Ok(()) + }) + }) + }; + let isolation = |options: PgPoolOptions| { + options.after_connect(|conn, _meta| { + Box::pin(async move { + let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") + .fetch_one(conn) + .await?; + if isolation != "read committed" { + return Err(sqlx::Error::Configuration( + format!("writer pool requires READ COMMITTED transaction isolation, got {isolation}").into(), + )); + } + Ok(()) + }) + }) + }; + match mode { + WriterHookMode::Combined => PgPoolOptions::new().after_connect(|conn, _meta| { + Box::pin(async move { + sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") + .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) + .execute(&mut *conn) + .await?; + let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") + .fetch_one(&mut *conn) + .await?; + if isolation != "read committed" { + return Err(sqlx::Error::Configuration( + format!("writer pool requires READ COMMITTED transaction isolation, got {isolation}").into(), + )); + } + Ok(()) + }) + }), + WriterHookMode::Separate => isolation(floor(PgPoolOptions::new())), + WriterHookMode::CombinedWithoutIsolation => floor(PgPoolOptions::new()), + WriterHookMode::None => PgPoolOptions::new(), + } +} + /// Configuration for the Postgres connection pool. #[derive(Debug, Clone)] pub struct DbConfig { @@ -8383,6 +8446,82 @@ mod tests { drop_scratch_db(&admin, pool, &name).await; } + async fn hook_mode_observation( + url: &str, + mode: WriterHookMode, + ) -> std::result::Result<(Option, String), sqlx::Error> { + let connect: PgConnectOptions = url.parse()?; + let pool = writer_pool_options_for_mode(mode) + .max_connections(1) + .min_connections(1) + .acquire_timeout(Duration::from_secs(1)) + .connect_with(connect) + .await?; + let floor: Option = + sqlx::query_scalar("SELECT NULLIF(current_setting('buzz.created_at_floor', true), '')") + .fetch_one(&pool) + .await?; + let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") + .fetch_one(&pool) + .await?; + pool.close().await; + Ok((floor, isolation)) + } + + #[tokio::test] + #[ignore = "requires Postgres"] + async fn writer_hook_modes_distinguish_floor_and_isolation_verdicts() { + let admin = PgPool::connect(&admin_url().await) + .await + .expect("connect admin"); + let (pool, name) = create_scratch_db(&admin, "hook_modes").await; + pool.close().await; + let base = admin_url().await; + let idx = base.rfind('/').expect("database URL path"); + let url = format!("{}/{}", &base[..idx], name); + + let (floor, isolation) = hook_mode_observation(&url, WriterHookMode::Combined) + .await + .expect("combined hook"); + assert_eq!( + floor, + Some(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) + ); + assert_eq!(isolation, "read committed"); + + let (floor, isolation) = hook_mode_observation(&url, WriterHookMode::Separate) + .await + .expect("separate hooks"); + assert_eq!( + floor, None, + "last after_connect registration must replace floor setup" + ); + assert_eq!(isolation, "read committed"); + + let (floor, isolation) = + hook_mode_observation(&url, WriterHookMode::CombinedWithoutIsolation) + .await + .expect("floor-only hook"); + assert_eq!( + floor, + Some(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) + ); + assert_eq!(isolation, "read committed"); + + let (floor, isolation) = hook_mode_observation(&url, WriterHookMode::None) + .await + .expect("no-hook control"); + assert_eq!(floor, None); + assert_eq!(isolation, "read committed"); + + sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE {name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop hook mode database"); + } + #[test] fn writer_pool_safety_hook_is_single_and_composed() { let source = include_str!("lib.rs"); From cb6ea7ffe3c8b429d7fb219046d4bfb61efa98b2 Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 20:49:12 -0700 Subject: [PATCH 39/51] test: bind dynamic sql fingerprints Co-authored-by: cid Signed-off-by: cid --- .../buzz-db/tests/community_fenced_writes.rs | 331 ++++++++++++------ .../bad_dynamic_second_in_owner.rs | 6 + .../bad_querybuilder_alias.rs | 5 + .../bad_querybuilder_branch.rs | 8 + .../bad_querybuilder_two_receivers.rs | 6 + scripts/lints/community_conditional_sql.yml | 6 + scripts/lints/community_const_sql.yml | 6 + .../community_querybuilder_mutations.yml | 12 +- 8 files changed, 272 insertions(+), 108 deletions(-) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_second_in_owner.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_alias.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_branch.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_two_receivers.rs create mode 100644 scripts/lints/community_conditional_sql.yml create mode 100644 scripts/lints/community_const_sql.yml diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index b3b2615423..6341a70a9f 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -13,114 +13,102 @@ use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; const AUTHORITY: &[&str] = buzz_db::deletion::EXPECTED_SCOPED_TABLES; -const NAMED_SQL_WRITES: &[(&str, &str, &str)] = &[( +const NAMED_SQL: &[(&str, &str, &str)] = &[( "crates/buzz-db/src/reaction.rs", "ADD_REACTION_SQL", - "ON CONFLICT (community_id, event_created_at, event_id, pubkey, emoji)", + "ADD_REACTION_SQL", )]; -const CONDITIONAL_SQL_WRITES: &[(&str, &str, &[&str])] = &[( +const CONDITIONAL_SQL: &[(&str, &str, &str)] = &[( "crates/buzz-db/src/lib.rs", - "let statement = if hard_delete_superseded", - &[ - "DELETE FROM events", - "UPDATE events SET deleted_at = NOW()", - "WHERE community_id = $1 AND kind = $2 AND pubkey = $3 AND d_tag = $4", - ], + "replace_parameterized_event", + "statement", )]; -const DYNAMIC_SQL: &[(&str, &str, &[&str])] = &[ +const DYNAMIC_SQL: &[(&str, &str, &str)] = &[ ( "crates/buzz-push-gateway/src/postgres.rs", "apply_migrations_and_grants", - &[ - "REVOKE CREATE ON DATABASE {database}", - "GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE", - ], + "AssertSqlSafe(grants)", ), ( "crates/buzz-db/src/channel.rs", "get_accessible_channels", - &["FROM channels c", "WHERE c.community_id = $1"], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/channel.rs", "get_users_bulk", - &["FROM users WHERE community_id = $1"], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/channel.rs", "update_channel", - &[ - "UPDATE channels SET {}", - "WHERE community_id = ${param_idx}", - ], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/deletion.rs", "inventory_schema", - &["SELECT count(*)::BIGINT FROM {table} WHERE community_id = $1"], + "AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/deletion.rs", "purge_postgres", - &["DELETE FROM {table} WHERE community_id = $1"], + "AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/deletion.rs", "verify_postgres_logically_deleted", - &["SELECT EXISTS(SELECT 1 FROM {table} WHERE community_id = $1 LIMIT 1)"], - ), - ( - "crates/buzz-db/src/lib.rs", - "insert_mentions", - &[ - "INSERT INTO event_mentions", - "push_bind(community_id.as_uuid())", - ], + "AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/partition.rs", "ensure_partition", - &["CREATE TABLE IF NOT EXISTS {partition_name} PARTITION OF {table_name}"], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/replica_fence.rs", "reader_supports_aurora_identity", - &["SELECT {AURORA_IDENTITY_FN}()"], + r#"sqlx::AssertSqlSafe(format!("SELECT {AURORA_IDENTITY_FN}()"))"#, ), ( "crates/buzz-db/src/replica_fence.rs", "observe_heartbeat", - &["FROM replica_heartbeat WHERE id = 1"], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/thread.rs", "get_thread_replies_on", - &["WHERE tm.community_id = $1"], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/thread.rs", "get_channel_window_on", - &["WHERE e.community_id = $1"], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/usage.rs", "active_user_counts", - &["SELECT", "FROM events e"], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/usage.rs", "active_channel_counts", - &["SELECT community_id", "FROM events"], + "sqlx::AssertSqlSafe(sql)", ), ( "crates/buzz-db/src/user.rs", "update_user_profile", - &["UPDATE users SET {}", "WHERE community_id = ${param_idx}"], + "sqlx::AssertSqlSafe(sql)", ), ]; +const QUERY_BUILDER_EXCEPTIONS: &[(&str, &str)] = &[ + ("crates/buzz-db/src/lib.rs", "insert_mentions"), + ("crates/buzz-db/src/event.rs", "query_events_on"), + ("crates/buzz-db/src/event.rs", "count_events_on"), +]; + // PostgreSQL syntax intentionally unsupported by sqlparser 0.62. Each entry is // an exact, reviewed statement fingerprint; source edits fail closed until the // statement either parses or this inventory is deliberately updated. @@ -284,6 +272,18 @@ fn function_owners(roots: &[PathBuf]) -> Vec { .collect() } +fn const_sql_definitions(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule(roots, "scripts/lints/community_const_sql.yml") +} + +fn conditional_sql_definitions(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule(roots, "scripts/lints/community_conditional_sql.yml") +} + +fn normalize_token(token: &str) -> String { + token.chars().filter(|ch| !ch.is_whitespace()).collect() +} + fn rust_string_value(token: &str) -> Option { if let Ok(literal) = litrs::StringLit::parse(token) { return Some(literal.value().to_owned()); @@ -559,6 +559,8 @@ fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec Vec Vec Vec Vec { let owners = function_owners(roots); let mutations = querybuilder_mutations(roots); + let builds = querybuilder_builds(roots); let mut found = Vec::new(); - for build in querybuilder_builds(roots) { + for build in &builds { let path = PathBuf::from(build["file"].as_str().expect("builder file")); let offset = build["range"]["byteOffset"]["start"] .as_u64() .expect("builder offset"); - let owner = owners + let Some(owner) = owners .iter() - .find(|owner| owner.path == path && owner.start <= offset && offset < owner.end); - let Some(owner) = owner else { continue }; + .find(|owner| owner.path == path && owner.start <= offset && offset < owner.end) + else { + continue; + }; if !owner.text.contains("QueryBuilder") { continue; } let receiver = build["metaVariables"]["single"]["QB"]["text"] .as_str() .expect("builder receiver"); - let fragments = mutations + if !receiver + .chars() + .all(|ch| ch == '_' || ch.is_ascii_alphanumeric()) + { + found.push(format!( + "{}: QueryBuilder build has non-identifier receiver {receiver:?}", + path.display() + )); + continue; + } + let mut owned = mutations .iter() .filter(|mutation| { - path == Path::new(mutation["file"].as_str().unwrap_or_default()) + Path::new(mutation["file"].as_str().unwrap_or_default()) == path && mutation["range"]["byteOffset"]["start"] .as_u64() .is_some_and(|start| owner.start <= start && start < owner.end) + && mutation["metaVariables"]["single"] + .get("QB") + .and_then(|value| value["text"].as_str()) + == Some(receiver) }) - .filter_map(|mutation| mutation["metaVariables"]["single"].get("SQL")?["text"].as_str()) - .filter_map(rust_string_value) .collect::>(); - if fragments.is_empty() { + owned.sort_by_key(|mutation| { + mutation["range"]["byteOffset"]["start"] + .as_u64() + .unwrap_or_default() + }); + let constructors = owned + .iter() + .filter(|mutation| { + mutation["text"] + .as_str() + .is_some_and(|text| text.starts_with("let mut ")) + }) + .count(); + if constructors != 1 { + let relative = path + .strip_prefix(repo_root()) + .unwrap_or(&path) + .to_string_lossy(); + let excepted = QUERY_BUILDER_EXCEPTIONS + .iter() + .any(|(file, expected_owner)| relative == *file && owner.name == *expected_owner); + if !excepted { + found.push(format!("{}: QueryBuilder {receiver} must have exactly one direct constructor in owner {}; got {constructors}", path.display(), owner.name)); + } + continue; + } + if owner.text.contains(&format!("let mut alias = {receiver}")) + || owner.text.contains(&format!("let alias = {receiver}")) + || owner.text.contains(&format!("{receiver} =")) && constructors > 1 + { found.push(format!( - "{}: QueryBuilder {receiver} has no literal fragments", + "{}: QueryBuilder {receiver} has ambiguous alias/reassignment", path.display() )); continue; } + let fragments = owned + .iter() + .filter_map(|mutation| mutation["metaVariables"]["single"].get("SQL")?["text"].as_str()) + .map(|token| rust_string_value(token).ok_or_else(|| token.to_owned())) + .collect::, _>>(); + let fragments = match fragments { + Ok(fragments) => fragments, + Err(token) => { + found.push(format!( + "{}: QueryBuilder {receiver} has nonliteral fragment {token}", + path.display() + )); + continue; + } + }; let reconstructed = fragments.join(" "); if !["insert into", "update ", "delete from"] .iter() @@ -758,13 +875,13 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { let statements = match parse_sql(&reconstructed) { Ok(statements) => statements, Err(error) => { - let inventoried = DYNAMIC_SQL.iter().any(|(file, expected_owner, guards)| { - relative == *file - && owner.name == *expected_owner - && guards.iter().all(|guard| owner.text.contains(guard)) - }); + let inventoried = QUERY_BUILDER_EXCEPTIONS + .iter() + .any(|(file, expected_owner)| { + relative == *file && owner.name == *expected_owner + }); if !inventoried { - found.push(format!("{}: mutating QueryBuilder {receiver} failed reconstruction and has no exact owner inventory: {error}; fragments={fragments:?}", path.display())); + found.push(format!("{}: mutating QueryBuilder {receiver} failed reconstruction and has no exact inventory: {error}; fragments={fragments:?}", path.display())); } continue; } @@ -858,6 +975,10 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { ("bad_query_as_with", "query_as_with constructor"), ("bad_query_scalar_with", "query_scalar_with constructor"), ("bad_raw_sql", "raw_sql constructor"), + ( + "bad_dynamic_second_in_owner", + "second dynamic expression in inventoried owner", + ), ] { assert!( violations @@ -867,7 +988,13 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { ); } let builder_violations = querybuilder_violations(std::slice::from_ref(&fixture)); - for fixture_name in ["bad_querybuilder_direct", "bad_querybuilder_split"] { + for fixture_name in [ + "bad_querybuilder_direct", + "bad_querybuilder_split", + "bad_querybuilder_two_receivers", + "bad_querybuilder_branch", + "bad_querybuilder_alias", + ] { assert!( builder_violations .iter() diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_second_in_owner.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_second_in_owner.rs new file mode 100644 index 0000000000..dcb9bdeb8e --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_second_in_owner.rs @@ -0,0 +1,6 @@ +fn update_channel() { + let sql = "UPDATE channels SET name=$1 WHERE community_id=$2".to_string(); + sqlx::query(sqlx::AssertSqlSafe(sql)); + let attacker_sql = "DELETE FROM relay_invites WHERE expires_at < now()".to_string(); + sqlx::query::(sqlx::AssertSqlSafe(attacker_sql)); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_alias.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_alias.rs new file mode 100644 index 0000000000..b0d5e502d0 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_alias.rs @@ -0,0 +1,5 @@ +fn mutant() { + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at = if flag { + sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE community_id=$1") + } else { + sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE community_id=$1"); + let mut unsafe_q: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at::new($SQL) - - pattern: sqlx::QueryBuilder::new($SQL) - - pattern: sqlx::QueryBuilder::<$$$T>::new($SQL) - - pattern: $QB.push($SQL) + - pattern: 'let mut $QB: $TYPE = QueryBuilder::new($SQL);' + - pattern: 'let mut $QB = QueryBuilder::new($SQL);' + - pattern: 'let mut $QB: $TYPE = sqlx::QueryBuilder::new($SQL);' + - pattern: 'let mut $QB = sqlx::QueryBuilder::new($SQL);' + - pattern: '$QB.push($SQL)' From 64be86a48823446e80c14ce82853d0e307122c42 Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 20:59:38 -0700 Subject: [PATCH 40/51] test: exercise unsafe hook defaults Co-authored-by: cid Signed-off-by: cid --- crates/buzz-db/src/lib.rs | 59 +++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 6f3cc3461b..937e680345 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -8474,13 +8474,13 @@ mod tests { let admin = PgPool::connect(&admin_url().await) .await .expect("connect admin"); - let (pool, name) = create_scratch_db(&admin, "hook_modes").await; - pool.close().await; let base = admin_url().await; let idx = base.rfind('/').expect("database URL path"); - let url = format!("{}/{}", &base[..idx], name); - let (floor, isolation) = hook_mode_observation(&url, WriterHookMode::Combined) + let (safe_pool, safe_name) = create_scratch_db(&admin, "hook_modes_safe").await; + safe_pool.close().await; + let safe_url = format!("{}/{}", &base[..idx], safe_name); + let (floor, isolation) = hook_mode_observation(&safe_url, WriterHookMode::Combined) .await .expect("combined hook"); assert_eq!( @@ -8488,8 +8488,7 @@ mod tests { Some(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) ); assert_eq!(isolation, "read committed"); - - let (floor, isolation) = hook_mode_observation(&url, WriterHookMode::Separate) + let (floor, isolation) = hook_mode_observation(&safe_url, WriterHookMode::Separate) .await .expect("separate hooks"); assert_eq!( @@ -8497,9 +8496,8 @@ mod tests { "last after_connect registration must replace floor setup" ); assert_eq!(isolation, "read committed"); - let (floor, isolation) = - hook_mode_observation(&url, WriterHookMode::CombinedWithoutIsolation) + hook_mode_observation(&safe_url, WriterHookMode::CombinedWithoutIsolation) .await .expect("floor-only hook"); assert_eq!( @@ -8507,19 +8505,56 @@ mod tests { Some(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) ); assert_eq!(isolation, "read committed"); - - let (floor, isolation) = hook_mode_observation(&url, WriterHookMode::None) + let (floor, isolation) = hook_mode_observation(&safe_url, WriterHookMode::None) .await .expect("no-hook control"); assert_eq!(floor, None); assert_eq!(isolation, "read committed"); + let (unsafe_pool, unsafe_name) = create_scratch_db(&admin, "hook_modes_unsafe").await; + unsafe_pool.close().await; sqlx::query(sqlx::AssertSqlSafe(format!( - "DROP DATABASE {name} WITH (FORCE)" + "ALTER DATABASE {unsafe_name} SET default_transaction_isolation = 'repeatable read'" ))) .execute(&admin) .await - .expect("drop hook mode database"); + .expect("set unsafe default"); + let unsafe_url = format!("{}/{}", &base[..idx], unsafe_name); + assert!( + hook_mode_observation(&unsafe_url, WriterHookMode::Combined) + .await + .is_err(), + "combined hook must reject unsafe isolation" + ); + assert!( + hook_mode_observation(&unsafe_url, WriterHookMode::Separate) + .await + .is_err(), + "separate last isolation hook rejects but still lacks floor on safe arm" + ); + let (floor, isolation) = + hook_mode_observation(&unsafe_url, WriterHookMode::CombinedWithoutIsolation) + .await + .expect("floor-only mutant admits unsafe default"); + assert_eq!( + floor, + Some(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) + ); + assert_eq!(isolation, "repeatable read"); + let (floor, isolation) = hook_mode_observation(&unsafe_url, WriterHookMode::None) + .await + .expect("no-hook mutant admits unsafe default"); + assert_eq!(floor, None); + assert_eq!(isolation, "repeatable read"); + + for name in [safe_name, unsafe_name] { + sqlx::query(sqlx::AssertSqlSafe(format!( + "DROP DATABASE {name} WITH (FORCE)" + ))) + .execute(&admin) + .await + .expect("drop hook mode database"); + } } #[test] From 513cf7499356df7e308dd6553c9adf62b87580ea Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 21:46:00 -0700 Subject: [PATCH 41/51] test: bind scanner to execution state Co-authored-by: cid Signed-off-by: cid --- .../buzz-db/tests/community_fenced_writes.rs | 348 +++++++++++++----- .../bad_querybuilder_post_build_push.rs | 6 + 2 files changed, 253 insertions(+), 101 deletions(-) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_post_build_push.rs diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index 6341a70a9f..22ad82ad11 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -5,6 +5,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; +use sha2::{Digest, Sha256}; use sqlparser::ast::{ BinaryOperator, Expr, FromTable, FunctionArg, FunctionArgExpr, FunctionArguments, Insert, ObjectName, Query, Select, SelectItem, SetExpr, Statement, TableFactor, TableObject, Value, @@ -25,82 +26,108 @@ const CONDITIONAL_SQL: &[(&str, &str, &str)] = &[( "statement", )]; -const DYNAMIC_SQL: &[(&str, &str, &str)] = &[ - ( - "crates/buzz-push-gateway/src/postgres.rs", - "apply_migrations_and_grants", - "AssertSqlSafe(grants)", - ), - ( - "crates/buzz-db/src/channel.rs", - "get_accessible_channels", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/channel.rs", - "get_users_bulk", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/channel.rs", - "update_channel", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/deletion.rs", - "inventory_schema", - "AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/deletion.rs", - "purge_postgres", - "AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/deletion.rs", - "verify_postgres_logically_deleted", - "AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/partition.rs", - "ensure_partition", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/replica_fence.rs", - "reader_supports_aurora_identity", - r#"sqlx::AssertSqlSafe(format!("SELECT {AURORA_IDENTITY_FN}()"))"#, - ), - ( - "crates/buzz-db/src/replica_fence.rs", - "observe_heartbeat", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/thread.rs", - "get_thread_replies_on", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/thread.rs", - "get_channel_window_on", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/usage.rs", - "active_user_counts", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/usage.rs", - "active_channel_counts", - "sqlx::AssertSqlSafe(sql)", - ), - ( - "crates/buzz-db/src/user.rs", - "update_user_profile", - "sqlx::AssertSqlSafe(sql)", - ), +#[derive(Clone, Copy)] +struct DynamicSqlInventory { + file: &'static str, + owner: &'static str, + argument: &'static str, + /// SHA-256 of the complete owning function source. + /// This binds a dynamic execution to the SQL-producing dataflow, not merely + /// to a local identifier such as `sql` at the execution call. + owner_fingerprint: &'static str, +} + +const DYNAMIC_SQL: &[DynamicSqlInventory] = &[ + DynamicSqlInventory { + file: "crates/buzz-push-gateway/src/postgres.rs", + owner: "apply_migrations_and_grants", + argument: "AssertSqlSafe(grants)", + owner_fingerprint: "59f2bddd81022c88a9b0367cde36ec53e07c44a09ad643d0db5f3dd83d5244e0", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/channel.rs", + owner: "get_accessible_channels", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "7af8e9f780718222d78e758c31195886532537b7182169b1a72b6f983d8fb5c9", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/channel.rs", + owner: "get_users_bulk", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "ed904badf2c3b9dfeaf6e0c4fe560753732a1b1fcdd6428c8c4f0d27a19c29cb", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/channel.rs", + owner: "update_channel", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "ec7c073f7e618b4d69c79d093b1c8a8320aeb3c8464f550c96a1c0b33162841f", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/deletion.rs", + owner: "inventory_schema", + argument: "AssertSqlSafe(sql)", + owner_fingerprint: "99d744cf8d197e25ee7b620763e7959923517ea7b6f9b5374a0a1c512b5abafb", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/deletion.rs", + owner: "purge_postgres", + argument: "AssertSqlSafe(sql)", + owner_fingerprint: "5d4334785028afef39f9d2f0c42aafce6052a880aeb02d72cd1f102390f5605c", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/deletion.rs", + owner: "verify_postgres_logically_deleted", + argument: "AssertSqlSafe(sql)", + owner_fingerprint: "1faf00caabcd24464fd6d5a83c3006f9e1b4cdc232dfbe9b66c9efbf7b6e7983", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/partition.rs", + owner: "ensure_partition", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "c2802a7c0fec6293a887befe9f78e4becb76b83b1d7315c1e01496bfb5d3b1ff", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/replica_fence.rs", + owner: "reader_supports_aurora_identity", + argument: r#"sqlx::AssertSqlSafe(format!("SELECT {AURORA_IDENTITY_FN}()"))"#, + owner_fingerprint: "3291240355f9b2901a9ec016a4049bccee71dd89fb4b888c8e95573f9519679e", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/replica_fence.rs", + owner: "observe_heartbeat", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "2f7628e054c9e2a195ed92b01994152150d23ea1a4b87b00e30cdaf99b4d829f", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/thread.rs", + owner: "get_thread_replies_on", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "265ee1d19c2b353857518b25185189993e6ee1b729623890c017aeea2dac2721", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/thread.rs", + owner: "get_channel_window_on", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "084c3d9e3cc3621b8602e7d6a653c8af320152a6bac752d95c57e72fb749e4d4", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/usage.rs", + owner: "active_user_counts", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "2fe366035e0eacd5c071e544d81dab8fdcc220e2904b779ec193b95484122571", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/usage.rs", + owner: "active_channel_counts", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "c812cd6eabca88997c6a46933dfcd45a28360716d1aff35086f7ac02cfbdc94a", + }, + DynamicSqlInventory { + file: "crates/buzz-db/src/user.rs", + owner: "update_user_profile", + argument: "sqlx::AssertSqlSafe(sql)", + owner_fingerprint: "2f2be400087028ede8d9281ea5c2b073dbd06f08ef628116f6c3160ddfe3990d", + }, ]; const QUERY_BUILDER_EXCEPTIONS: &[(&str, &str)] = &[ @@ -281,7 +308,30 @@ fn conditional_sql_definitions(roots: &[PathBuf]) -> Vec { } fn normalize_token(token: &str) -> String { - token.chars().filter(|ch| !ch.is_whitespace()).collect() + token + .chars() + .filter(|ch| !ch.is_ascii_whitespace()) + .collect() +} + +fn source_fingerprint(source: &str) -> String { + let normalized = source + .bytes() + .filter(|byte| !byte.is_ascii_whitespace()) + .collect::>(); + hex::encode(Sha256::digest(normalized)) +} + +fn dynamic_inventory_matches( + entry: &DynamicSqlInventory, + relative: &str, + owner: &FunctionOwner, + sql_token: &str, +) -> bool { + relative == entry.file + && owner.name == entry.owner + && normalize_token(sql_token) == normalize_token(entry.argument) + && source_fingerprint(&owner.text) == entry.owner_fingerprint } fn rust_string_value(token: &str) -> Option { @@ -554,11 +604,19 @@ fn parse_sql(sql: &str) -> Result, String> { } fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec { + let owners = function_owners(roots); + inspect_matches_with_owners(matches, roots, &owners) +} + +fn inspect_matches_with_owners( + matches: &[serde_json::Value], + roots: &[PathBuf], + owners: &[FunctionOwner], +) -> Vec { let fenced = authority(); let root = repo_root(); let mut violations = Vec::new(); let mut observed_parser_exceptions = BTreeSet::new(); - let owners = function_owners(roots); let const_defs = const_sql_definitions(roots); let conditional_defs = conditional_sql_definitions(roots); let test_ranges = test_module_ranges(roots); @@ -608,11 +666,7 @@ fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec Vec Vec Vec { let offset = build["range"]["byteOffset"]["start"] .as_u64() .expect("builder offset"); - let Some(owner) = owners + let owner = owners .iter() - .find(|owner| owner.path == path && owner.start <= offset && offset < owner.end) - else { + .find(|owner| owner.path == path && owner.start <= offset && offset < owner.end); + let Some(owner) = owner else { continue; }; if !owner.text.contains("QueryBuilder") { @@ -797,7 +857,7 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { )); continue; } - let mut owned = mutations + let mut all_owned = mutations .iter() .filter(|mutation| { Path::new(mutation["file"].as_str().unwrap_or_default()) == path @@ -810,12 +870,12 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { == Some(receiver) }) .collect::>(); - owned.sort_by_key(|mutation| { + all_owned.sort_by_key(|mutation| { mutation["range"]["byteOffset"]["start"] .as_u64() .unwrap_or_default() }); - let constructors = owned + let constructors = all_owned .iter() .filter(|mutation| { mutation["text"] @@ -846,8 +906,29 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { )); continue; } - let fragments = owned + let constructor_offset = all_owned .iter() + .find(|mutation| { + mutation["text"] + .as_str() + .is_some_and(|text| text.starts_with("let mut ")) + }) + .and_then(|mutation| mutation["range"]["byteOffset"]["start"].as_u64()) + .expect("exactly one QueryBuilder constructor"); + if constructor_offset >= offset { + found.push(format!( + "{}: QueryBuilder {receiver} constructor does not precede build", + path.display() + )); + continue; + } + let fragments = all_owned + .iter() + .filter(|mutation| { + mutation["range"]["byteOffset"]["start"] + .as_u64() + .is_some_and(|start| constructor_offset <= start && start < offset) + }) .filter_map(|mutation| mutation["metaVariables"]["single"].get("SQL")?["text"].as_str()) .map(|token| rust_string_value(token).ok_or_else(|| token.to_owned())) .collect::, _>>(); @@ -899,6 +980,70 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { found } +#[test] +fn scanner_rejects_dynamic_producer_changes_with_the_same_argument_token() { + const REVIEWED_PRODUCER: &str = r#""UPDATE channels SET {}, updated_at = NOW() WHERE community_id = ${param_idx} AND id = ${channel_param_idx} AND deleted_at IS NULL""#; + const UNSAFE_PRODUCER: &str = r#""DELETE FROM relay_invites WHERE expires_at < now() /* {} ${param_idx} ${channel_param_idx} */""#; + + let roots = production_roots(); + let matches = ast_matches(&roots); + let mut owners = function_owners(&roots); + let entry = DYNAMIC_SQL + .iter() + .find(|entry| { + entry.file == "crates/buzz-db/src/channel.rs" && entry.owner == "update_channel" + }) + .expect("update_channel dynamic inventory"); + let owner_index = owners + .iter() + .position(|owner| owner.path == repo_root().join(entry.file) && owner.name == entry.owner) + .expect("update_channel owner"); + let source = owners[owner_index].text.clone(); + assert!( + source.contains(REVIEWED_PRODUCER), + "reviewed update_channel SQL producer drifted" + ); + assert!( + source.contains(entry.argument), + "calibration requires the inventoried argument token" + ); + + for (description, mutant) in [ + ( + "unsafe SQL producer", + source.replacen(REVIEWED_PRODUCER, UNSAFE_PRODUCER, 1), + ), + ( + "additional dynamic execution in the reviewed owner", + source.replacen( + entry.argument, + &format!( + "{}); sqlx::query(sqlx::AssertSqlSafe(attacker_sql", + entry.argument + ), + 1, + ), + ), + ] { + assert!( + mutant.contains(entry.argument), + "{description} mutant changed the inventoried SQL argument token" + ); + owners[owner_index].text = mutant; + let violations = inspect_matches_with_owners(&matches, &roots, &owners); + assert!( + violations.iter().any(|violation| { + violation.contains("crates/buzz-db/src/channel.rs") + && violation.contains( + "nonliteral SQL expression must match exactly one inventory entry", + ) + && violation.contains(entry.argument) + }), + "{description} mutant escaped: {violations:?}" + ); + } +} + #[test] fn every_workspace_sql_execution_is_scanned_and_querybuilders_are_owned() { let roots = production_roots(); @@ -994,6 +1139,7 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { "bad_querybuilder_two_receivers", "bad_querybuilder_branch", "bad_querybuilder_alias", + "bad_querybuilder_post_build_push", ] { assert!( builder_violations diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_post_build_push.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_post_build_push.rs new file mode 100644 index 0000000000..0a24219046 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_post_build_push.rs @@ -0,0 +1,6 @@ +fn mutant() { + let mut qb: sqlx::QueryBuilder = + sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at < now()"); + let _ = qb.build(); + qb.push(" AND community_id = $1"); +} From 95927504c8b9266e8af7390caa64f114caf9608b Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 22:02:10 -0700 Subject: [PATCH 42/51] test: reject conditional mutating builders Co-authored-by: cid Signed-off-by: cid --- .../buzz-db/tests/community_fenced_writes.rs | 58 +++++++++++++++++++ .../bad_querybuilder_conditional_push.rs | 8 +++ .../good_querybuilder_conditional_select.rs | 8 +++ .../community_querybuilder_control_flow.yml | 13 +++++ 4 files changed, 87 insertions(+) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/good_querybuilder_conditional_select.rs create mode 100644 scripts/lints/community_querybuilder_control_flow.yml diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index 22ad82ad11..d4b7491487 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -236,6 +236,13 @@ fn querybuilder_builds(roots: &[PathBuf]) -> Vec { ast_matches_for_rule(roots, "scripts/lints/community_querybuilder_builds.yml") } +fn querybuilder_control_flow(roots: &[PathBuf]) -> Vec { + ast_matches_for_rule( + roots, + "scripts/lints/community_querybuilder_control_flow.yml", + ) +} + fn test_module_ranges(roots: &[PathBuf]) -> BTreeMap> { let mut cfg_offsets = BTreeMap::>::new(); for matched in ast_matches_for_rule(roots, "scripts/lints/community_cfg_test_attributes.yml") { @@ -829,6 +836,7 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { let owners = function_owners(roots); let mutations = querybuilder_mutations(roots); let builds = querybuilder_builds(roots); + let control_flow = querybuilder_control_flow(roots); let mut found = Vec::new(); for build in &builds { let path = PathBuf::from(build["file"].as_str().expect("builder file")); @@ -949,6 +957,42 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { { continue; } + // Reconstruct first so read-only builders retain ordinary conditional + // filters. Mutating builders fail closed when any contributing state or + // the build itself is nested under branch/loop/closure control flow. + let owner_control_flow = control_flow.iter().filter(|control| { + Path::new(control["file"].as_str().unwrap_or_default()) == path + && control["range"]["byteOffset"]["start"] + .as_u64() + .zip(control["range"]["byteOffset"]["end"].as_u64()) + .is_some_and(|(start, end)| owner.start <= start && end <= owner.end) + }); + let mut state_offsets = all_owned + .iter() + .filter_map(|mutation| { + mutation["range"]["byteOffset"]["start"] + .as_u64() + .filter(|mutation_offset| { + constructor_offset <= *mutation_offset && *mutation_offset < offset + }) + }) + .collect::>(); + state_offsets.push(offset); + let build_state_is_conditional = state_offsets.into_iter().any(|state_offset| { + owner_control_flow.clone().any(|control| { + control["range"]["byteOffset"]["start"] + .as_u64() + .zip(control["range"]["byteOffset"]["end"].as_u64()) + .is_some_and(|(start, end)| start <= state_offset && state_offset < end) + }) + }); + if build_state_is_conditional { + found.push(format!( + "{}: mutating QueryBuilder {receiver} constructor, mutation, or build is control-flow-dependent", + path.display() + )); + continue; + } let relative = path .strip_prefix(repo_root()) .unwrap_or(&path) @@ -1139,6 +1183,7 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { "bad_querybuilder_two_receivers", "bad_querybuilder_branch", "bad_querybuilder_alias", + "bad_querybuilder_conditional_push", "bad_querybuilder_post_build_push", ] { assert!( @@ -1148,6 +1193,19 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { "QueryBuilder control {fixture_name} escaped: {builder_violations:?}" ); } + assert!( + builder_violations.iter().any(|violation| { + violation.contains("bad_querybuilder_conditional_push") + && violation.contains("control-flow-dependent") + }), + "conditional QueryBuilder tenant predicate escaped the control-flow gate: {builder_violations:?}" + ); + assert!( + !builder_violations + .iter() + .any(|violation| violation.contains("good_querybuilder_conditional_select")), + "conditional SELECT QueryBuilder was rejected: {builder_violations:?}" + ); assert!( !violations.iter().any(|v| v.contains("good_tenant_delete")), "tenant-bound control was rejected: {violations:?}" diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs new file mode 100644 index 0000000000..1247dc3903 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs @@ -0,0 +1,8 @@ +fn mutant(scope_to_tenant: bool) { + let mut qb: sqlx::QueryBuilder = + sqlx::QueryBuilder::new("DELETE FROM relay_invites"); + if scope_to_tenant { + qb.push(" AND community_id = $1"); + } + let _ = qb.build(); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_querybuilder_conditional_select.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_querybuilder_conditional_select.rs new file mode 100644 index 0000000000..d164955f73 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_querybuilder_conditional_select.rs @@ -0,0 +1,8 @@ +fn control(filter_by_kind: bool) { + let mut qb: sqlx::QueryBuilder = + sqlx::QueryBuilder::new("SELECT id FROM events WHERE community_id = $1"); + if filter_by_kind { + qb.push(" AND kind = $2"); + } + let _ = qb.build(); +} diff --git a/scripts/lints/community_querybuilder_control_flow.yml b/scripts/lints/community_querybuilder_control_flow.yml new file mode 100644 index 0000000000..7397cfbda7 --- /dev/null +++ b/scripts/lints/community_querybuilder_control_flow.yml @@ -0,0 +1,13 @@ +id: community-querybuilder-control-flow +message: Control-flow region used to reject ambiguous QueryBuilder state +severity: hint +language: rust +rule: + any: + - kind: if_expression + - kind: for_expression + - kind: while_expression + - kind: loop_expression + - kind: match_expression + - kind: closure_expression + - kind: async_block From a7caa1ad335110b33d1536522e6c39ede0199a33 Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 22:07:25 -0700 Subject: [PATCH 43/51] test: reconstruct builder sql exactly Co-authored-by: cid Signed-off-by: cid --- crates/buzz-db/tests/community_fenced_writes.rs | 6 ++++-- .../bad_querybuilder_conditional_push.rs | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index d4b7491487..7f5cacccb7 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -950,7 +950,9 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { continue; } }; - let reconstructed = fragments.join(" "); + // QueryBuilder appends string fragments exactly; inserting separators + // would let a split `DE` + `LETE` mutation evade classification. + let reconstructed = fragments.concat(); if !["insert into", "update ", "delete from"] .iter() .any(|keyword| reconstructed.to_ascii_lowercase().contains(keyword)) @@ -1198,7 +1200,7 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { violation.contains("bad_querybuilder_conditional_push") && violation.contains("control-flow-dependent") }), - "conditional QueryBuilder tenant predicate escaped the control-flow gate: {builder_violations:?}" + "split-keyword conditional QueryBuilder tenant predicate escaped the control-flow gate: {builder_violations:?}" ); assert!( !builder_violations diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs index 1247dc3903..95f80b7716 100644 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs @@ -1,6 +1,6 @@ fn mutant(scope_to_tenant: bool) { - let mut qb: sqlx::QueryBuilder = - sqlx::QueryBuilder::new("DELETE FROM relay_invites"); + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DE"); + qb.push("LETE FROM relay_invites"); if scope_to_tenant { qb.push(" AND community_id = $1"); } From 32c373eade9903b8be09e7f9de4a48cf0b8183e8 Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 22:42:53 -0700 Subject: [PATCH 44/51] test: classify builder mutations by sql ast Co-authored-by: Kalvin Chau Signed-off-by: Kalvin Chau --- .../buzz-db/tests/community_fenced_writes.rs | 162 ++++++++++++++---- .../bad_querybuilder_conditional_push.rs | 6 +- ...bad_querybuilder_conditional_whitespace.rs | 18 ++ 3 files changed, 154 insertions(+), 32 deletions(-) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index 7f5cacccb7..32f9d3c0ce 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -130,10 +130,51 @@ const DYNAMIC_SQL: &[DynamicSqlInventory] = &[ }, ]; -const QUERY_BUILDER_EXCEPTIONS: &[(&str, &str)] = &[ - ("crates/buzz-db/src/lib.rs", "insert_mentions"), - ("crates/buzz-db/src/event.rs", "query_events_on"), - ("crates/buzz-db/src/event.rs", "count_events_on"), +// Exact owners whose QueryBuilder classification cannot be proved from the +// extracted literal fragments alone. Their complete owner source is pinned so +// each exception becomes stale on any edit. +struct QueryBuilderException { + file: &'static str, + owner: &'static str, + owner_fingerprint: &'static str, +} + +const QUERY_BUILDER_EXCEPTIONS: &[QueryBuilderException] = &[ + QueryBuilderException { + file: "crates/buzz-db/src/lib.rs", + owner: "insert_mentions", + owner_fingerprint: "c888a340801e900d6511de81c5d63c6f87afa030a87847b9df8b23846904b213", + }, + QueryBuilderException { + file: "crates/buzz-db/src/event.rs", + owner: "query_events_on", + owner_fingerprint: "f06a482fa31252f376ca6cec4a83e9e2675f052810e5d8bbfe6784a7dd44dc45", + }, + QueryBuilderException { + file: "crates/buzz-db/src/event.rs", + owner: "count_events_on", + owner_fingerprint: "7f48326ae5789c421653ed85fc612242e9a7b054ec9915cda13b4d294dd107f2", + }, + QueryBuilderException { + file: "crates/buzz-db/src/event.rs", + owner: "get_last_message_at_bulk", + owner_fingerprint: "a3e9ef0f2eea7df90f052380e85fa7f96cd80ac540969d8ab8319fb135d98312", + }, + QueryBuilderException { + file: "crates/buzz-db/src/event.rs", + owner: "get_events_by_ids_on", + owner_fingerprint: "6f18d34658052e2437767c7ff8063cf3e28dfc11a5b8a2c001c329dc1114a056", + }, + QueryBuilderException { + file: "crates/buzz-db/src/channel.rs", + owner: "get_member_counts_bulk", + owner_fingerprint: "877c8e955a81bd28b8477a4967b67e3036ff95be8e7c3e33a4413119b2cc58aa", + }, + QueryBuilderException { + file: "crates/buzz-search/src/query.rs", + owner: "search", + owner_fingerprint: "ca8adf19af50a6c7310bc2e9d1c7ed7be000677815e9a2fe46af4b158409f189", + }, ]; // PostgreSQL syntax intentionally unsupported by sqlparser 0.62. Each entry is @@ -341,6 +382,16 @@ fn dynamic_inventory_matches( && source_fingerprint(&owner.text) == entry.owner_fingerprint } +fn querybuilder_exception_matches( + entry: &QueryBuilderException, + relative: &str, + owner: &FunctionOwner, +) -> bool { + relative == entry.file + && owner.name == entry.owner + && source_fingerprint(&owner.text) == entry.owner_fingerprint +} + fn rust_string_value(token: &str) -> Option { if let Ok(literal) = litrs::StringLit::parse(token) { return Some(literal.value().to_owned()); @@ -606,6 +657,20 @@ fn mutation_violation(statement: &Statement, fenced: &BTreeSet) -> Optio } } +fn querybuilder_mutation_statement(statement: &Statement) -> bool { + matches!( + statement, + Statement::Delete(_) | Statement::Update(_) | Statement::Insert(_) + ) +} + +fn parsed_querybuilder_is_read_only(statements: &[Statement]) -> bool { + !statements.is_empty() + && statements + .iter() + .all(|statement| matches!(statement, Statement::Query(_))) +} + fn parse_sql(sql: &str) -> Result, String> { Parser::parse_sql(&PostgreSqlDialect {}, sql).map_err(|error| error.to_string()) } @@ -833,11 +898,15 @@ fn inspect_matches_with_owners( } fn querybuilder_violations(roots: &[PathBuf]) -> Vec { + let production_scan = roots + .iter() + .any(|root| root.starts_with(repo_root().join("crates")) && root.ends_with("src")); let owners = function_owners(roots); let mutations = querybuilder_mutations(roots); let builds = querybuilder_builds(roots); let control_flow = querybuilder_control_flow(roots); let mut found = Vec::new(); + let mut observed_querybuilder_exceptions = BTreeSet::new(); for build in &builds { let path = PathBuf::from(build["file"].as_str().expect("builder file")); let offset = build["range"]["byteOffset"]["start"] @@ -898,8 +967,10 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { .to_string_lossy(); let excepted = QUERY_BUILDER_EXCEPTIONS .iter() - .any(|(file, expected_owner)| relative == *file && owner.name == *expected_owner); - if !excepted { + .find(|entry| querybuilder_exception_matches(entry, &relative, owner)); + if let Some(entry) = excepted { + observed_querybuilder_exceptions.insert((entry.file, entry.owner)); + } else { found.push(format!("{}: QueryBuilder {receiver} must have exactly one direct constructor in owner {}; got {constructors}", path.display(), owner.name)); } continue; @@ -950,18 +1021,33 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { continue; } }; - // QueryBuilder appends string fragments exactly; inserting separators - // would let a split `DE` + `LETE` mutation evade classification. + // QueryBuilder appends string fragments exactly, so parse the same + // byte sequence that will be executed. The AST, not raw keyword + // spelling, decides whether conditional state is safe to ignore. let reconstructed = fragments.concat(); - if !["insert into", "update ", "delete from"] + let relative = path + .strip_prefix(repo_root()) + .unwrap_or(&path) + .to_string_lossy(); + let inventory_entry = QUERY_BUILDER_EXCEPTIONS .iter() - .any(|keyword| reconstructed.to_ascii_lowercase().contains(keyword)) - { + .find(|entry| querybuilder_exception_matches(entry, &relative, owner)); + if let Some(entry) = inventory_entry { + observed_querybuilder_exceptions.insert((entry.file, entry.owner)); + } + let inventoried = inventory_entry.is_some(); + let parsed = parse_sql(&reconstructed); + let is_read_only = parsed + .as_deref() + .is_ok_and(parsed_querybuilder_is_read_only); + // Fingerprinted exceptions cover builders whose dynamic bind helpers + // prevent a complete literal reconstruction, not parsed mutations. + if is_read_only || (inventoried && parsed.is_err()) { continue; } - // Reconstruct first so read-only builders retain ordinary conditional - // filters. Mutating builders fail closed when any contributing state or - // the build itself is nested under branch/loop/closure control flow. + // Parsed reads and fingerprinted, unparseable exact-owner exceptions + // are safe to ignore here. Every other shape must pass the conditional- + // state gate and AST mutation validation below. let owner_control_flow = control_flow.iter().filter(|control| { Path::new(control["file"].as_str().unwrap_or_default()) == path && control["range"]["byteOffset"]["start"] @@ -995,21 +1081,17 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { )); continue; } - let relative = path - .strip_prefix(repo_root()) - .unwrap_or(&path) - .to_string_lossy(); - let statements = match parse_sql(&reconstructed) { - Ok(statements) => statements, + let statements = match parsed { + Ok(statements) if statements.iter().all(querybuilder_mutation_statement) => statements, + Ok(statements) => { + found.push(format!( + "{}: QueryBuilder {receiver} is neither read-only nor exclusively mutating and has no exact inventory: statements={statements:?}; fragments={fragments:?}", + path.display() + )); + continue; + } Err(error) => { - let inventoried = QUERY_BUILDER_EXCEPTIONS - .iter() - .any(|(file, expected_owner)| { - relative == *file && owner.name == *expected_owner - }); - if !inventoried { - found.push(format!("{}: mutating QueryBuilder {receiver} failed reconstruction and has no exact inventory: {error}; fragments={fragments:?}", path.display())); - } + found.push(format!("{}: QueryBuilder {receiver} failed reconstruction and has no exact inventory: {error}; fragments={fragments:?}", path.display())); continue; } }; @@ -1023,6 +1105,16 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { }) })); } + if production_scan { + for entry in QUERY_BUILDER_EXCEPTIONS { + if !observed_querybuilder_exceptions.contains(&(entry.file, entry.owner)) { + found.push(format!( + "{}: QueryBuilder inventory entry for {} is stale or unobserved", + entry.file, entry.owner + )); + } + } + } found } @@ -1186,6 +1278,7 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { "bad_querybuilder_branch", "bad_querybuilder_alias", "bad_querybuilder_conditional_push", + "bad_querybuilder_conditional_whitespace", "bad_querybuilder_post_build_push", ] { assert!( @@ -1200,7 +1293,18 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { violation.contains("bad_querybuilder_conditional_push") && violation.contains("control-flow-dependent") }), - "split-keyword conditional QueryBuilder tenant predicate escaped the control-flow gate: {builder_violations:?}" + "comment-separated conditional DELETE QueryBuilder tenant predicate escaped the control-flow gate: {builder_violations:?}" + ); + assert!( + builder_violations + .iter() + .filter(|violation| { + violation.contains("bad_querybuilder_conditional_whitespace") + && violation.contains("control-flow-dependent") + }) + .count() + == 2, + "tab-separated UPDATE or newline-separated INSERT QueryBuilder escaped the control-flow gate: {builder_violations:?}" ); assert!( !builder_violations diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs index 95f80b7716..9c453fc4ae 100644 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs @@ -1,8 +1,8 @@ fn mutant(scope_to_tenant: bool) { - let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DE"); - qb.push("LETE FROM relay_invites"); + let mut qb: sqlx::QueryBuilder = + sqlx::QueryBuilder::new("DELETE/**/FROM relay_invites"); if scope_to_tenant { - qb.push(" AND community_id = $1"); + qb.push(" WHERE community_id = $1"); } let _ = qb.build(); } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs new file mode 100644 index 0000000000..7891fdceda --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs @@ -0,0 +1,18 @@ +fn tab_separated_update(scope_to_tenant: bool) { + let mut qb: sqlx::QueryBuilder = + sqlx::QueryBuilder::new("UPDATE\trelay_invites SET expires_at = now()"); + if scope_to_tenant { + qb.push(" WHERE community_id = $1"); + } + let _ = qb.build(); +} + +fn newline_separated_insert(scope_to_tenant: bool) { + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( + "INSERT\nINTO relay_invites (community_id, token_hash) VALUES ($1, $2)", + ); + if scope_to_tenant { + qb.push(" ON CONFLICT DO NOTHING"); + } + let _ = qb.build(); +} From 6407af1c6d0914a9a843a78a83e2251f2cc35eeb Mon Sep 17 00:00:00 2001 From: cid Date: Wed, 5 Aug 2026 23:03:17 -0700 Subject: [PATCH 45/51] test: recurse through builder mutation ast Co-authored-by: Kalvin Chau Signed-off-by: Kalvin Chau --- Cargo.lock | 12 +++ crates/buzz-db/Cargo.toml | 2 +- .../buzz-db/tests/community_fenced_writes.rs | 79 ++++++++++++++----- ...bad_querybuilder_conditional_whitespace.rs | 32 ++++++++ .../bad_querybuilder_unconditional_cte.rs | 27 +++++++ 5 files changed, 132 insertions(+), 20 deletions(-) create mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_unconditional_cte.rs diff --git a/Cargo.lock b/Cargo.lock index 3309e83b28..0757919c3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8969,6 +8969,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" dependencies = [ "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", ] [[package]] diff --git a/crates/buzz-db/Cargo.toml b/crates/buzz-db/Cargo.toml index f1aa618d44..f18b105874 100644 --- a/crates/buzz-db/Cargo.toml +++ b/crates/buzz-db/Cargo.toml @@ -26,7 +26,7 @@ metrics = { workspace = true } [dev-dependencies] tokio = { workspace = true } metrics-util = { workspace = true } -sqlparser = { version = "0.62", default-features = false, features = ["std"] } +sqlparser = { version = "0.62", default-features = false, features = ["std", "visitor"] } walkdir = "2" litrs = "1" regex-lite = "0.1" diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs index 32f9d3c0ce..ff385ee46f 100644 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ b/crates/buzz-db/tests/community_fenced_writes.rs @@ -7,8 +7,9 @@ use std::process::Command; use sha2::{Digest, Sha256}; use sqlparser::ast::{ - BinaryOperator, Expr, FromTable, FunctionArg, FunctionArgExpr, FunctionArguments, Insert, - ObjectName, Query, Select, SelectItem, SetExpr, Statement, TableFactor, TableObject, Value, + visit_statements, BinaryOperator, Expr, FromTable, FunctionArg, FunctionArgExpr, + FunctionArguments, Insert, ObjectName, Query, Select, SelectItem, SetExpr, Statement, + TableFactor, TableObject, Value, }; use sqlparser::dialect::PostgreSqlDialect; use sqlparser::parser::Parser; @@ -657,18 +658,41 @@ fn mutation_violation(statement: &Statement, fenced: &BTreeSet) -> Optio } } -fn querybuilder_mutation_statement(statement: &Statement) -> bool { +fn statement_is_mutation(statement: &Statement) -> bool { matches!( statement, Statement::Delete(_) | Statement::Update(_) | Statement::Insert(_) ) } -fn parsed_querybuilder_is_read_only(statements: &[Statement]) -> bool { +fn mutation_violations( + statements: &Vec, + fenced: &BTreeSet, +) -> (usize, Vec) { + let mut mutation_count = 0; + let mut violations = Vec::new(); + let traversal = visit_statements(statements, |statement| { + if statement_is_mutation(statement) { + mutation_count += 1; + if let Some(violation) = mutation_violation(statement, fenced) { + violations.push(violation); + } + } + std::ops::ControlFlow::<()>::Continue(()) + }); + assert!(matches!(traversal, std::ops::ControlFlow::Continue(()))); + (mutation_count, violations) +} + +fn parsed_querybuilder_is_read_only( + statements: &Vec, + fenced: &BTreeSet, +) -> bool { !statements.is_empty() && statements .iter() .all(|statement| matches!(statement, Statement::Query(_))) + && mutation_violations(statements, fenced).0 == 0 } fn parse_sql(sql: &str) -> Result, String> { @@ -1037,9 +1061,10 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { } let inventoried = inventory_entry.is_some(); let parsed = parse_sql(&reconstructed); + let fenced = authority(); let is_read_only = parsed - .as_deref() - .is_ok_and(parsed_querybuilder_is_read_only); + .as_ref() + .is_ok_and(|statements| parsed_querybuilder_is_read_only(statements, &fenced)); // Fingerprinted exceptions cover builders whose dynamic bind helpers // prevent a complete literal reconstruction, not parsed mutations. if is_read_only || (inventoried && parsed.is_err()) { @@ -1082,10 +1107,10 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { continue; } let statements = match parsed { - Ok(statements) if statements.iter().all(querybuilder_mutation_statement) => statements, + Ok(statements) if mutation_violations(&statements, &fenced).0 > 0 => statements, Ok(statements) => { found.push(format!( - "{}: QueryBuilder {receiver} is neither read-only nor exclusively mutating and has no exact inventory: statements={statements:?}; fragments={fragments:?}", + "{}: QueryBuilder {receiver} is neither recursively read-only nor mutation-bearing and has no exact inventory: statements={statements:?}; fragments={fragments:?}", path.display() )); continue; @@ -1095,15 +1120,17 @@ fn querybuilder_violations(roots: &[PathBuf]) -> Vec { continue; } }; - let fenced = authority(); - found.extend(statements.iter().filter_map(|statement| { - mutation_violation(statement, &fenced).map(|violation| { - format!( - "{}: mutating QueryBuilder {receiver}: {violation}", - path.display() - ) - }) - })); + found.extend( + mutation_violations(&statements, &fenced) + .1 + .into_iter() + .map(|violation| { + format!( + "{}: mutating QueryBuilder {receiver}: {violation}", + path.display() + ) + }), + ); } if production_scan { for entry in QUERY_BUILDER_EXCEPTIONS { @@ -1279,6 +1306,7 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { "bad_querybuilder_alias", "bad_querybuilder_conditional_push", "bad_querybuilder_conditional_whitespace", + "bad_querybuilder_unconditional_cte", "bad_querybuilder_post_build_push", ] { assert!( @@ -1303,8 +1331,21 @@ fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { && violation.contains("control-flow-dependent") }) .count() - == 2, - "tab-separated UPDATE or newline-separated INSERT QueryBuilder escaped the control-flow gate: {builder_violations:?}" + == 5, + "tab/newline direct DML or conditional DELETE/UPDATE/INSERT CTE QueryBuilder escaped the control-flow gate: {builder_violations:?}" + ); + assert!( + builder_violations + .iter() + .filter(|violation| { + violation.contains("bad_querybuilder_unconditional_cte") + && (violation.contains("fleet DELETE") + || violation.contains("fleet UPDATE") + || violation.contains("omits community_id")) + }) + .count() + == 4, + "unconditional or nested DELETE/UPDATE/INSERT CTE QueryBuilder escaped recursive tenant validation: {builder_violations:?}" ); assert!( !builder_violations diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs index 7891fdceda..dc6d7af1c2 100644 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs @@ -16,3 +16,35 @@ fn newline_separated_insert(scope_to_tenant: bool) { } let _ = qb.build(); } + +fn conditional_delete_cte(scope_to_tenant: bool) { + let mut qb: sqlx::QueryBuilder = + sqlx::QueryBuilder::new("WITH gone AS (DELETE FROM relay_invites"); + if scope_to_tenant { + qb.push(" WHERE community_id = $1"); + } + qb.push(" RETURNING 1) SELECT 1"); + let _ = qb.build(); +} + +fn conditional_update_cte(scope_to_tenant: bool) { + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( + "WITH changed AS (UPDATE relay_invites SET expires_at = now()", + ); + if scope_to_tenant { + qb.push(" WHERE community_id = $1"); + } + qb.push(" RETURNING 1) SELECT 1"); + let _ = qb.build(); +} + +fn conditional_insert_cte(scope_to_tenant: bool) { + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( + "WITH made AS (INSERT INTO relay_invites (community_id, token_hash)", + ); + if scope_to_tenant { + qb.push(" VALUES ($1, $2)"); + } + qb.push(" RETURNING 1) SELECT 1"); + let _ = qb.build(); +} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_unconditional_cte.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_unconditional_cte.rs new file mode 100644 index 0000000000..ec17a088c4 --- /dev/null +++ b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_unconditional_cte.rs @@ -0,0 +1,27 @@ +fn unconditional_delete_cte() { + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( + "WITH gone AS (DELETE FROM relay_invites RETURNING 1) SELECT 1", + ); + let _ = qb.build(); +} + +fn unconditional_update_cte() { + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( + "WITH changed AS (UPDATE relay_invites SET expires_at = now() RETURNING 1) SELECT 1", + ); + let _ = qb.build(); +} + +fn unconditional_insert_cte() { + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( + "WITH made AS (INSERT INTO relay_invites (token_hash) VALUES ($1) RETURNING 1) SELECT 1", + ); + let _ = qb.build(); +} + +fn nested_unconditional_delete_cte() { + let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( + "WITH outer_cte AS (WITH gone AS (DELETE FROM relay_invites RETURNING 1) SELECT 1) SELECT 1", + ); + let _ = qb.build(); +} From 32b61c7f76dc7d64dd6dec63f082a656ab149fb9 Mon Sep 17 00:00:00 2001 From: cid Date: Thu, 6 Aug 2026 00:52:24 -0700 Subject: [PATCH 46/51] fix: restore bounded deletion retries Co-authored-by: Kalvin Chau Signed-off-by: Kalvin Chau --- crates/buzz-db/src/deletion.rs | 112 +++++++++++++++++++++++-- crates/buzz-deletion/src/lib.rs | 2 + migrations/0028_community_deletion.sql | 4 + schema/schema.sql | 4 + 4 files changed, 116 insertions(+), 6 deletions(-) diff --git a/crates/buzz-db/src/deletion.rs b/crates/buzz-db/src/deletion.rs index b094c292d5..34dc896bdc 100644 --- a/crates/buzz-db/src/deletion.rs +++ b/crates/buzz-db/src/deletion.rs @@ -218,6 +218,8 @@ pub struct DeletionRequest { pub community_host: String, /// Current lifecycle stage. pub stage: DeletionStage, + /// Stage at which the current consecutive retry streak started. + pub retry_stage: Option, /// Operator identity that submitted the request. pub requested_by: String, /// Optional request reason. @@ -242,10 +244,12 @@ pub struct DeletionRequest { pub lease_until: Option>, /// Number of claims. pub attempts: i32, - /// Number of failed execution units. + /// Number of consecutive failed execution attempts at `retry_stage`. pub retry_count: i32, /// Last bounded error. pub last_error: Option, + /// Earliest time a transient failure may be claimed again. + pub next_attempt_at: DateTime, /// Permanent fail-closed block reason. pub blocked_reason: Option, /// Submission time. @@ -1729,7 +1733,8 @@ impl DeletionStore { let affected = sqlx::query( "UPDATE community_deletion_requests \ SET stage = 'retention_pending', completed_at = now(), updated_at = now(), \ - lease_owner = NULL, lease_until = NULL, last_error = NULL, last_error_at = NULL \ + lease_owner = NULL, lease_until = NULL, retry_count = 0, retry_stage = NULL, \ + last_error = NULL, last_error_at = NULL \ WHERE id = $1 AND stage = 'logically_verified' \ AND lease_owner = $2 AND lease_generation = $3 AND lease_until >= now() \ AND fence_generation = $4", @@ -1749,6 +1754,10 @@ impl DeletionStore { } /// Persist a retryable unit failure and release the claim. + /// + /// The eighth consecutive failure at the same stage becomes a durable + /// block. A successful stage transition clears the streak; an operator may + /// use [`Self::unblock`] after remediating an exhausted dependency failure. pub async fn record_retry( &self, token: &LeaseToken, @@ -1761,11 +1770,27 @@ impl DeletionStore { let retry_seconds = i64::try_from(retry_after.as_secs()).unwrap_or(i64::MAX); let mut tx = self.pool.begin().await?; verify_lease(&mut tx, token, stage).await?; + let (retry_count, retry_stage): (i32, Option) = sqlx::query_as( + "SELECT retry_count, retry_stage FROM community_deletion_requests WHERE id = $1 FOR UPDATE", + ) + .bind(token.request_id) + .fetch_one(&mut *tx) + .await?; + let stage_name = stage.to_string(); + let consecutive_retries = if retry_stage.as_deref() == Some(stage_name.as_str()) { + retry_count.saturating_add(1) + } else { + 1 + }; + let exhausted = consecutive_retries >= 8; checkpoint_failed_tx(&mut tx, token, stage, unit_key, &bounded).await?; sqlx::query( "UPDATE community_deletion_requests \ - SET retry_count = retry_count + 1, last_error = $4, last_error_at = now(), \ - next_attempt_at = now() + make_interval(secs => $5), \ + SET retry_count = $7, retry_stage = $8, last_error = $4, last_error_at = now(), \ + next_attempt_at = CASE WHEN $6 THEN next_attempt_at \ + ELSE now() + make_interval(secs => $5) END, \ + blocked_at = CASE WHEN $6 THEN now() ELSE blocked_at END, \ + blocked_reason = CASE WHEN $6 THEN $4 ELSE blocked_reason END, \ lease_owner = NULL, lease_until = NULL, updated_at = now() \ WHERE id = $1 AND lease_owner = $2 AND lease_generation = $3", ) @@ -1774,6 +1799,9 @@ impl DeletionStore { .bind(token.generation) .bind(&bounded) .bind(retry_seconds) + .bind(exhausted) + .bind(consecutive_retries) + .bind(stage_name) .execute(&mut *tx) .await?; tx.commit().await?; @@ -1849,7 +1877,7 @@ impl DeletionStore { let row = sqlx::query( "UPDATE community_deletion_requests \ - SET blocked_at = NULL, blocked_reason = NULL, retry_count = 0, \ + SET blocked_at = NULL, blocked_reason = NULL, retry_count = 0, retry_stage = NULL, \ last_error = NULL, last_error_at = NULL, next_attempt_at = now(), \ lease_owner = NULL, lease_until = NULL, updated_at = now() \ WHERE id = $1 AND blocked_at IS NOT NULL RETURNING *", @@ -2453,7 +2481,8 @@ async fn advance_request_tx( let affected = sqlx::query( "UPDATE community_deletion_requests \ SET stage = $5, fence_generation = COALESCE($6, fence_generation), \ - updated_at = now(), last_error = NULL, last_error_at = NULL \ + updated_at = now(), retry_count = 0, retry_stage = NULL, \ + last_error = NULL, last_error_at = NULL \ WHERE id = $1 AND stage = $4 AND lease_owner = $2 \ AND lease_generation = $3 AND lease_until >= now() AND blocked_at IS NULL", ) @@ -2547,6 +2576,10 @@ fn row_to_request(row: sqlx::postgres::PgRow) -> Result { community_id: CommunityId::from_uuid(community_id), community_host: row.try_get("community_host")?, stage: row.try_get::("stage")?.parse()?, + retry_stage: row + .try_get::, _>("retry_stage")? + .map(|stage| stage.parse()) + .transpose()?, requested_by: row.try_get("requested_by")?, reason: row.try_get("reason")?, schema_manifest: row.try_get("schema_manifest")?, @@ -2561,6 +2594,7 @@ fn row_to_request(row: sqlx::postgres::PgRow) -> Result { attempts: row.try_get("attempts")?, retry_count: row.try_get("retry_count")?, last_error: row.try_get("last_error")?, + next_attempt_at: row.try_get("next_attempt_at")?, blocked_reason: row.try_get("blocked_reason")?, created_at: row.try_get("created_at")?, updated_at: row.try_get("updated_at")?, @@ -3023,6 +3057,72 @@ mod postgres_tests { .is_none()); } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn retry_exhaustion_blocks_only_the_consecutive_stage_and_progress_resets_it() { + let (db, store) = store().await; + let (request, _) = inventoried_request(&db, &store).await; + store + .approve(request.id, "approver", None) + .await + .expect("approve"); + + for attempt in 1..=8 { + let claim = store + .claim_specific( + request.id, + &format!("executor-{attempt}"), + DEFAULT_LEASE_DURATION, + ) + .await + .expect("claim retryable request") + .expect("request remains claimable before exhaustion"); + store + .record_retry( + &claim.lease, + DeletionStage::Approved, + "dependency", + "dependency unavailable", + Duration::ZERO, + ) + .await + .expect("record retry"); + + let observed = store.get(request.id).await.expect("load retry state"); + assert_eq!(observed.retry_count, attempt); + assert_eq!(observed.retry_stage, Some(DeletionStage::Approved)); + assert_eq!(observed.blocked_reason.is_some(), attempt == 8); + } + assert!(store + .claim_specific(request.id, "blocked-executor", DEFAULT_LEASE_DURATION) + .await + .expect("claim blocked request") + .is_none()); + + let recovered = store + .unblock(request.id, "operator", "dependency repaired") + .await + .expect("unblock exhausted request"); + assert_eq!(recovered.retry_count, 0); + assert_eq!(recovered.retry_stage, None); + assert!(recovered.blocked_reason.is_none()); + + let claim = store + .claim_specific(request.id, "successor", DEFAULT_LEASE_DURATION) + .await + .expect("claim recovered request") + .expect("recovered request is claimable"); + store + .begin_quiescing(&claim.lease) + .await + .expect("begin quiescing after recovery"); + store.fence(&claim.lease).await.expect("advance stage"); + let advanced = store.get(request.id).await.expect("load advanced request"); + assert_eq!(advanced.stage, DeletionStage::Fenced); + assert_eq!(advanced.retry_count, 0); + assert_eq!(advanced.retry_stage, None); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn operator_unblock_preserves_approval_and_records_recovery() { diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index 2fa4dbaee3..ca9ef79f8c 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -348,6 +348,7 @@ struct RunOutput { stage: DeletionStage, retry_count: i32, last_error: Option, + next_attempt_at: chrono::DateTime, blocked_reason: Option, } @@ -1349,6 +1350,7 @@ fn run_output(request: DeletionRequest) -> RunOutput { stage: request.stage, retry_count: request.retry_count, last_error: request.last_error, + next_attempt_at: request.next_attempt_at, blocked_reason: request.blocked_reason, } } diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index 60737351c9..a140b35681 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -40,6 +40,10 @@ CREATE TABLE community_deletion_requests ( lease_until TIMESTAMPTZ, attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + retry_stage TEXT CHECK (retry_stage IS NULL OR retry_stage IN ( + 'approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified' + )), next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_error TEXT, last_error_at TIMESTAMPTZ, diff --git a/schema/schema.sql b/schema/schema.sql index 3e72f51e83..a094fea45a 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1151,6 +1151,10 @@ CREATE TABLE community_deletion_requests ( lease_until TIMESTAMPTZ, attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + retry_stage TEXT CHECK (retry_stage IS NULL OR retry_stage IN ( + 'approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified' + )), next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_error TEXT, last_error_at TIMESTAMPTZ, From 1cd15decf85b917804a20af0c6d9c521bec32bf3 Mon Sep 17 00:00:00 2001 From: cid Date: Thu, 6 Aug 2026 01:02:57 -0700 Subject: [PATCH 47/51] fix: migrate bounded retry state additively Co-authored-by: Kalvin Chau Signed-off-by: Kalvin Chau --- crates/buzz-db/src/migration.rs | 70 ++++++---------------- migrations/0028_community_deletion.sql | 4 -- migrations/0029_bound_deletion_retries.sql | 8 +++ 3 files changed, 25 insertions(+), 57 deletions(-) create mode 100644 migrations/0029_bound_deletion_retries.sql diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 1d6f699d01..9128e00135 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -668,13 +668,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); -<<<<<<< HEAD assert_eq!(migrations.len(), 29); -||||||| parent of 2ea419e57 (fix: advance community deletion migration to 0028) - assert_eq!(migrations.len(), 27); -======= - assert_eq!(migrations.len(), 28); ->>>>>>> 2ea419e57 (fix: advance community deletion migration to 0028) assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1033,27 +1027,6 @@ mod tests { assert!(heartbeat.contains("INSERT INTO replica_heartbeat (id) VALUES (1)")); assert!(heartbeat.contains("_operator_global_tables")); -<<<<<<< HEAD - - // Channel-id lookup index (0027). - let channel_id_index = migrations - .iter() - .find(|migration| migration.sql.as_str().contains("idx_channels_id_live")) - .expect("channel-id lookup migration"); - assert_eq!(channel_id_index.version, 27); - assert!(channel_id_index.sql.as_str().contains("INCLUDE (community_id)")); - assert!(channel_id_index.sql.as_str().contains("WHERE deleted_at IS NULL")); - assert!(!channel_id_index.sql.as_str().contains("CREATE UNIQUE INDEX")); -||||||| parent of 2ea419e57 (fix: advance community deletion migration to 0028) - - // Channel-id lookup index (0027): serves tenant-independent channel lookups. - assert_eq!(migrations[26].version, 27); - let channel_id_index = migrations[26].sql.as_str(); - assert!(channel_id_index.contains("idx_channels_id_live")); - assert!(channel_id_index.contains("INCLUDE (community_id)")); - assert!(channel_id_index.contains("WHERE deleted_at IS NULL")); - assert!(!channel_id_index.contains("CREATE UNIQUE INDEX")); -======= // Channel-id lookup index (0027): serves tenant-independent channel lookups. assert_eq!(migrations[26].version, 27); let channel_id_index = migrations[26].sql.as_str(); @@ -1061,34 +1034,12 @@ mod tests { assert!(channel_id_index.contains("INCLUDE (community_id)")); assert!(channel_id_index.contains("WHERE deleted_at IS NULL")); assert!(!channel_id_index.contains("CREATE UNIQUE INDEX")); ->>>>>>> 2ea419e57 (fix: advance community deletion migration to 0028) assert!(desired_schema.contains("idx_channels_id_live")); - // Main owns 0028 for long reaction payloads. - let long_reactions = migrations - .iter() - .find(|migration| { - migration - .sql - .as_str() - .contains("ALTER TABLE reactions ALTER COLUMN emoji TYPE VARCHAR(66)") - }) - .expect("long-reaction migration"); - assert_eq!(long_reactions.version, 28); - assert!(desired_schema.contains("emoji VARCHAR(66) NOT NULL")); - - // Durable whole-community deletion control plane. Final numbering is - // reconciled once all branch commits have replayed onto current main. - let deletion = migrations - .iter() - .find(|migration| { - migration - .sql - .as_str() - .contains("CREATE TABLE community_deletion_requests") - }) - .expect("community-deletion migration"); - let deletion = deletion.sql.as_str(); + // Durable whole-community deletion control plane and universal DB fence. + assert_eq!(migrations[27].version, 28); + let deletion = migrations[27].sql.as_str(); + assert!(deletion.contains("CREATE TABLE community_deletion_requests")); assert!(deletion.contains("CREATE TABLE community_deletion_approvals")); assert!(deletion.contains("CREATE TABLE community_deletion_checkpoints")); assert!(deletion.contains("CREATE TABLE community_serving_write_leases")); @@ -1104,10 +1055,23 @@ mod tests { assert!(deletion.contains("CREATE FUNCTION enforce_community_tombstone")); assert!(deletion.contains("community tombstones are permanent")); assert!(deletion.contains("SET LOCAL lock_timeout = '5s'")); + assert!(deletion.contains("'active', 'quiescing', 'fenced', 'tombstone'")); + assert!(deletion.contains("_operator_global_tables")); + assert!(deletion.contains("'submitted', 'inventoried', 'approved', 'fenced', 'drained'")); assert!(deletion.contains("UNIQUE (id, community_id, inventory_digest)")); assert!(deletion.contains("FOREIGN KEY (request_id, community_id, inventory_digest)")); assert!(deletion.contains("prevent_community_deletion_request_retargeting")); assert!(deletion.contains("prevent_community_deletion_approval_removal")); + + // Bounded deletion retries are an additive migration because 0028 is + // already checksum-pinned on deployed candidates. + assert_eq!(migrations[28].version, 29); + let bounded_retries = migrations[28].sql.as_str(); + assert!(bounded_retries.contains("ALTER TABLE community_deletion_requests")); + assert!(bounded_retries.contains("ADD COLUMN retry_stage")); + assert!(bounded_retries.contains("'approved', 'fenced', 'drained'")); + assert!(!deletion.contains("retry_stage")); + assert!(desired_schema.contains("retry_stage TEXT CHECK")); } #[test] diff --git a/migrations/0028_community_deletion.sql b/migrations/0028_community_deletion.sql index a140b35681..60737351c9 100644 --- a/migrations/0028_community_deletion.sql +++ b/migrations/0028_community_deletion.sql @@ -40,10 +40,6 @@ CREATE TABLE community_deletion_requests ( lease_until TIMESTAMPTZ, attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), - retry_stage TEXT CHECK (retry_stage IS NULL OR retry_stage IN ( - 'approved', 'fenced', 'drained', 'bindings_removed', - 'postgres_purged', 'cache_purged', 'logically_verified' - )), next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_error TEXT, last_error_at TIMESTAMPTZ, diff --git a/migrations/0029_bound_deletion_retries.sql b/migrations/0029_bound_deletion_retries.sql new file mode 100644 index 0000000000..8a6b9cbe4e --- /dev/null +++ b/migrations/0029_bound_deletion_retries.sql @@ -0,0 +1,8 @@ +-- Restore the per-stage retry streak needed to bound transient deletion +-- failures. Migration 0028 is already deployed and checksum-pinned, so this +-- schema addition must remain additive. +ALTER TABLE community_deletion_requests + ADD COLUMN retry_stage TEXT CHECK (retry_stage IS NULL OR retry_stage IN ( + 'approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified' + )); From 9a9db550ceff90f714f9f9e47742d01b443ffe74 Mon Sep 17 00:00:00 2001 From: cid Date: Thu, 6 Aug 2026 08:24:12 -0700 Subject: [PATCH 48/51] fix: ignore observational row drift before fencing Co-authored-by: Kalvin Chau Signed-off-by: Kalvin Chau --- crates/buzz-deletion/src/lib.rs | 43 ++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/crates/buzz-deletion/src/lib.rs b/crates/buzz-deletion/src/lib.rs index ca9ef79f8c..4fc38bac84 100644 --- a/crates/buzz-deletion/src/lib.rs +++ b/crates/buzz-deletion/src/lib.rs @@ -998,7 +998,9 @@ async fn execute_stage( .inventory_schema(request.community_id) .await?; let frozen = validate_frozen_inventory(request)?; - if live_schema != frozen.schema { + if live_schema.scoped_tables != frozen.schema.scoped_tables + || live_schema.fenced_tables != frozen.schema.fenced_tables + { return Err(permanent( "approved structural catalog drifted before fencing", )); @@ -1514,6 +1516,45 @@ mod tests { ) } + #[tokio::test] + #[ignore = "requires Postgres"] + async fn approved_stage_allows_post_inventory_row_churn_before_fencing() { + let (db, services, claim) = claimed_test_deletion("deletion-row-churn").await; + let frozen: FrozenInventory = serde_json::from_value( + claim + .request + .inventory_manifest + .clone() + .expect("frozen inventory"), + ) + .expect("decode frozen inventory"); + + db.add_to_allowlist(claim.request.community_id, &[0x41; 32], &[0x42; 32], None) + .await + .expect("post-inventory serving write"); + let live = services + .store + .inventory_schema(claim.request.community_id) + .await + .expect("live inventory after serving churn"); + assert_eq!(live.scoped_tables, frozen.schema.scoped_tables); + assert_eq!(live.fenced_tables, frozen.schema.fenced_tables); + assert_eq!( + live.row_counts["pubkey_allowlist"], + frozen.schema.row_counts["pubkey_allowlist"] + 1 + ); + + execute_stage(&services, &claim, &CancellationToken::new()) + .await + .expect("row-count churn must not fail structural revalidation"); + let fenced = services + .store + .get(claim.request.id) + .await + .expect("load fenced request"); + assert_eq!(fenced.stage, DeletionStage::Fenced); + } + #[tokio::test] #[ignore = "requires Postgres"] async fn frozen_inventory_digest_and_storage_ownership_fail_closed() { From 26a408283a305e72e7acaf9764d28b14e8050ab0 Mon Sep 17 00:00:00 2001 From: cid Date: Thu, 6 Aug 2026 09:19:04 -0700 Subject: [PATCH 49/51] chore: reconcile deletion migration numbering Co-authored-by: cid Signed-off-by: cid --- crates/buzz-db/src/migration.rs | 26 ++++++++++++------- ...letion.sql => 0029_community_deletion.sql} | 0 ...es.sql => 0030_bound_deletion_retries.sql} | 2 +- schema/schema.sql | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) rename migrations/{0028_community_deletion.sql => 0029_community_deletion.sql} (100%) rename migrations/{0029_bound_deletion_retries.sql => 0030_bound_deletion_retries.sql} (85%) diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 9128e00135..2c22a90e6b 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -501,7 +501,7 @@ mod tests { migrations.sort_by_key(|migration| migration.version); migrations .into_iter() - .filter(|migration| migration.version > 28) + .filter(|migration| migration.version > 29) .flat_map(|migration| { let statements = split_sql_statements(migration.sql.as_str()); let attachments = statements @@ -668,7 +668,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 29); + assert_eq!(migrations.len(), 30); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1036,9 +1036,17 @@ mod tests { assert!(!channel_id_index.contains("CREATE UNIQUE INDEX")); assert!(desired_schema.contains("idx_channels_id_live")); - // Durable whole-community deletion control plane and universal DB fence. + // Main owns 0028 for long reaction payloads. assert_eq!(migrations[27].version, 28); - let deletion = migrations[27].sql.as_str(); + let long_reactions = migrations[27].sql.as_str(); + assert!( + long_reactions.contains("ALTER TABLE reactions ALTER COLUMN emoji TYPE VARCHAR(66)") + ); + assert!(desired_schema.contains("emoji VARCHAR(66) NOT NULL")); + + // Durable whole-community deletion control plane and universal DB fence. + assert_eq!(migrations[28].version, 29); + let deletion = migrations[28].sql.as_str(); assert!(deletion.contains("CREATE TABLE community_deletion_requests")); assert!(deletion.contains("CREATE TABLE community_deletion_approvals")); assert!(deletion.contains("CREATE TABLE community_deletion_checkpoints")); @@ -1063,10 +1071,10 @@ mod tests { assert!(deletion.contains("prevent_community_deletion_request_retargeting")); assert!(deletion.contains("prevent_community_deletion_approval_removal")); - // Bounded deletion retries are an additive migration because 0028 is - // already checksum-pinned on deployed candidates. - assert_eq!(migrations[28].version, 29); - let bounded_retries = migrations[28].sql.as_str(); + // Bounded deletion retries are additive because the deletion migration + // is checksum-pinned once deployed. + assert_eq!(migrations[29].version, 30); + let bounded_retries = migrations[29].sql.as_str(); assert!(bounded_retries.contains("ALTER TABLE community_deletion_requests")); assert!(bounded_retries.contains("ADD COLUMN retry_stage")); assert!(bounded_retries.contains("'approved', 'fenced', 'drained'")); @@ -1079,7 +1087,7 @@ mod tests { let violations = migrations_missing_community_write_fence_attachment(); assert!( violations.is_empty(), - "migrations after 0028 must explicitly attach every new community write fence:\n{}", + "migrations after 0029 must explicitly attach every new community write fence:\n{}", violations.join("\n") ); } diff --git a/migrations/0028_community_deletion.sql b/migrations/0029_community_deletion.sql similarity index 100% rename from migrations/0028_community_deletion.sql rename to migrations/0029_community_deletion.sql diff --git a/migrations/0029_bound_deletion_retries.sql b/migrations/0030_bound_deletion_retries.sql similarity index 85% rename from migrations/0029_bound_deletion_retries.sql rename to migrations/0030_bound_deletion_retries.sql index 8a6b9cbe4e..88906536f5 100644 --- a/migrations/0029_bound_deletion_retries.sql +++ b/migrations/0030_bound_deletion_retries.sql @@ -1,5 +1,5 @@ -- Restore the per-stage retry streak needed to bound transient deletion --- failures. Migration 0028 is already deployed and checksum-pinned, so this +-- failures. Migration 0029 is already deployed and checksum-pinned, so this -- schema addition must remain additive. ALTER TABLE community_deletion_requests ADD COLUMN retry_stage TEXT CHECK (retry_stage IS NULL OR retry_stage IN ( diff --git a/schema/schema.sql b/schema/schema.sql index a094fea45a..61b766bbcd 100644 --- a/schema/schema.sql +++ b/schema/schema.sql @@ -1126,7 +1126,7 @@ INSERT INTO replica_heartbeat (id) VALUES (1); INSERT INTO _operator_global_tables (table_name, reason) VALUES ('replica_heartbeat', 'single-row replication freshness token; describes deployment topology, never tenant data'); --- ── Whole-community deletion control plane (migration 0028) ───────────────── +-- ── Whole-community deletion control plane (migration 0029) ───────────────── CREATE TABLE community_deletion_requests ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), community_id UUID NOT NULL UNIQUE REFERENCES communities(id), From 27aa6894f8af5710e2e9a00533823d6176cd84cf Mon Sep 17 00:00:00 2001 From: cid Date: Thu, 6 Aug 2026 09:46:53 -0700 Subject: [PATCH 50/51] chore: narrow community deletion scope Co-authored-by: cid Signed-off-by: cid --- Cargo.lock | 25 - Justfile | 6 +- bin/.ast-grep-0.39.7.pkg | 1 - bin/ast-grep | 1 - crates/buzz-db/Cargo.toml | 4 - crates/buzz-db/src/lib.rs | 174 --- crates/buzz-db/src/migration.rs | 149 +- .../buzz-db/tests/community_fenced_writes.rs | 1370 ----------------- .../bad_dynamic_assert_sql_safe.rs | 3 - .../bad_dynamic_second_in_owner.rs | 6 - .../bad_generic_query.rs | 3 - .../bad_insert_implicit_columns.rs | 3 - .../bad_insert_select.rs | 3 - .../bad_insert_unfenced_source.rs | 3 - .../bad_insert_unrelated_alias.rs | 3 - .../community_fenced_writes/bad_or_delete.rs | 3 - .../bad_production_mod_tests.rs | 5 - .../bad_query_as_with.rs | 1 - .../bad_query_scalar_with.rs | 1 - .../community_fenced_writes/bad_query_with.rs | 3 - .../bad_querybuilder_alias.rs | 5 - .../bad_querybuilder_branch.rs | 8 - .../bad_querybuilder_conditional_push.rs | 8 - ...bad_querybuilder_conditional_whitespace.rs | 50 - .../bad_querybuilder_direct.rs | 1 - .../bad_querybuilder_post_build_push.rs | 6 - .../bad_querybuilder_split.rs | 1 - .../bad_querybuilder_two_receivers.rs | 6 - .../bad_querybuilder_unconditional_cte.rs | 27 - .../community_fenced_writes/bad_raw_sql.rs | 3 - .../bad_unrelated_subquery.rs | 3 - .../good_cfg_test_module.rs | 6 - .../good_gated_delete.rs | 3 - .../good_querybuilder_conditional_select.rs | 8 - .../good_tenant_delete.rs | 3 - migrations/0029_community_deletion.sql | 4 + migrations/0030_bound_deletion_retries.sql | 8 - .../lints/community_cfg_test_attributes.yml | 7 - scripts/lints/community_conditional_sql.yml | 6 - scripts/lints/community_const_sql.yml | 6 - scripts/lints/community_fenced_writes.yml | 16 - scripts/lints/community_function_items.yml | 6 - scripts/lints/community_modules.yml | 6 - .../lints/community_querybuilder_builds.yml | 9 - .../community_querybuilder_control_flow.yml | 13 - .../community_querybuilder_creations.yml | 10 - .../community_querybuilder_mutations.yml | 11 - scripts/lints/community_sqlx_calls.yml | 16 - 48 files changed, 7 insertions(+), 2016 deletions(-) delete mode 120000 bin/.ast-grep-0.39.7.pkg delete mode 120000 bin/ast-grep delete mode 100644 crates/buzz-db/tests/community_fenced_writes.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_assert_sql_safe.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_second_in_owner.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_generic_query.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_implicit_columns.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_select.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unfenced_source.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unrelated_alias.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_or_delete.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_production_mod_tests.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_as_with.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_scalar_with.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_with.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_alias.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_branch.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_push.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_direct.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_post_build_push.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_split.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_two_receivers.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_unconditional_cte.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_raw_sql.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/bad_unrelated_subquery.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/good_cfg_test_module.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/good_gated_delete.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/good_querybuilder_conditional_select.rs delete mode 100644 crates/buzz-db/tests/fixtures/community_fenced_writes/good_tenant_delete.rs delete mode 100644 migrations/0030_bound_deletion_retries.sql delete mode 100644 scripts/lints/community_cfg_test_attributes.yml delete mode 100644 scripts/lints/community_conditional_sql.yml delete mode 100644 scripts/lints/community_const_sql.yml delete mode 100644 scripts/lints/community_fenced_writes.yml delete mode 100644 scripts/lints/community_function_items.yml delete mode 100644 scripts/lints/community_modules.yml delete mode 100644 scripts/lints/community_querybuilder_builds.yml delete mode 100644 scripts/lints/community_querybuilder_control_flow.yml delete mode 100644 scripts/lints/community_querybuilder_creations.yml delete mode 100644 scripts/lints/community_querybuilder_mutations.yml delete mode 100644 scripts/lints/community_sqlx_calls.yml diff --git a/Cargo.lock b/Cargo.lock index 0757919c3a..6e064b2e5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1006,22 +1006,18 @@ dependencies = [ "buzz-core", "chrono", "hex", - "litrs", "metrics", "metrics-util", "nostr", "rand 0.10.1", - "regex-lite", "serde", "serde_json", "sha2 0.11.0", - "sqlparser", "sqlx", "thiserror 2.0.18", "tokio", "tracing", "uuid", - "walkdir", ] [[package]] @@ -8962,27 +8958,6 @@ dependencies = [ "buzz-dev-mcp", ] -[[package]] -name = "sqlparser" -version = "0.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c6d1b651dc4edf07eead2a0c6c78016ce971bc2c10da5266861b13f25e7cec" -dependencies = [ - "log", - "sqlparser_derive", -] - -[[package]] -name = "sqlparser_derive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "sqlx" version = "0.9.0" diff --git a/Justfile b/Justfile index 8030d382dc..0a43249d5f 100644 --- a/Justfile +++ b/Justfile @@ -92,11 +92,7 @@ build-release: cargo build --workspace --release # Run repo lint and formatting checks -check: fmt-check community-fenced-writes-check clippy desktop-check desktop-tauri-fmt-check desktop-tauri-clippy web-check mobile-check - -# Structurally classify production SQL writes to community-fenced tables. -community-fenced-writes-check: - cargo test -p buzz-db --test community_fenced_writes +check: fmt-check clippy desktop-check desktop-tauri-fmt-check desktop-tauri-clippy web-check mobile-check # Format all Rust code fmt: diff --git a/bin/.ast-grep-0.39.7.pkg b/bin/.ast-grep-0.39.7.pkg deleted file mode 120000 index 383f4511d4..0000000000 --- a/bin/.ast-grep-0.39.7.pkg +++ /dev/null @@ -1 +0,0 @@ -hermit \ No newline at end of file diff --git a/bin/ast-grep b/bin/ast-grep deleted file mode 120000 index dd35c6a667..0000000000 --- a/bin/ast-grep +++ /dev/null @@ -1 +0,0 @@ -.ast-grep-0.39.7.pkg \ No newline at end of file diff --git a/crates/buzz-db/Cargo.toml b/crates/buzz-db/Cargo.toml index f18b105874..6f76a11bc1 100644 --- a/crates/buzz-db/Cargo.toml +++ b/crates/buzz-db/Cargo.toml @@ -26,7 +26,3 @@ metrics = { workspace = true } [dev-dependencies] tokio = { workspace = true } metrics-util = { workspace = true } -sqlparser = { version = "0.62", default-features = false, features = ["std", "visitor"] } -walkdir = "2" -litrs = "1" -regex-lite = "0.1" diff --git a/crates/buzz-db/src/lib.rs b/crates/buzz-db/src/lib.rs index 937e680345..e9ace6fd90 100644 --- a/crates/buzz-db/src/lib.rs +++ b/crates/buzz-db/src/lib.rs @@ -60,8 +60,6 @@ pub use error::{DbError, Result}; pub use event::{EventQuery, ReactionEventInsertOutcome, DEFAULT_MAX_PAGE_LIMIT}; use chrono::{DateTime, Utc}; -#[cfg(test)] -use sqlx::postgres::PgConnectOptions; use sqlx::postgres::{PgConnection, PgPoolOptions}; use sqlx::{Connection, PgPool, QueryBuilder, Row}; use std::time::Duration; @@ -502,67 +500,6 @@ impl UsageMetricsLeader { } } -#[cfg(test)] -#[derive(Clone, Copy)] -enum WriterHookMode { - Combined, - Separate, - CombinedWithoutIsolation, - None, -} - -#[cfg(test)] -fn writer_pool_options_for_mode(mode: WriterHookMode) -> PgPoolOptions { - let floor = |options: PgPoolOptions| { - options.after_connect(|conn, _meta| { - Box::pin(async move { - sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") - .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(conn) - .await?; - Ok(()) - }) - }) - }; - let isolation = |options: PgPoolOptions| { - options.after_connect(|conn, _meta| { - Box::pin(async move { - let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") - .fetch_one(conn) - .await?; - if isolation != "read committed" { - return Err(sqlx::Error::Configuration( - format!("writer pool requires READ COMMITTED transaction isolation, got {isolation}").into(), - )); - } - Ok(()) - }) - }) - }; - match mode { - WriterHookMode::Combined => PgPoolOptions::new().after_connect(|conn, _meta| { - Box::pin(async move { - sqlx::query("SELECT set_config('buzz.created_at_floor', $1, false)") - .bind(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - .execute(&mut *conn) - .await?; - let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") - .fetch_one(&mut *conn) - .await?; - if isolation != "read committed" { - return Err(sqlx::Error::Configuration( - format!("writer pool requires READ COMMITTED transaction isolation, got {isolation}").into(), - )); - } - Ok(()) - }) - }), - WriterHookMode::Separate => isolation(floor(PgPoolOptions::new())), - WriterHookMode::CombinedWithoutIsolation => floor(PgPoolOptions::new()), - WriterHookMode::None => PgPoolOptions::new(), - } -} - /// Configuration for the Postgres connection pool. #[derive(Debug, Clone)] pub struct DbConfig { @@ -8446,117 +8383,6 @@ mod tests { drop_scratch_db(&admin, pool, &name).await; } - async fn hook_mode_observation( - url: &str, - mode: WriterHookMode, - ) -> std::result::Result<(Option, String), sqlx::Error> { - let connect: PgConnectOptions = url.parse()?; - let pool = writer_pool_options_for_mode(mode) - .max_connections(1) - .min_connections(1) - .acquire_timeout(Duration::from_secs(1)) - .connect_with(connect) - .await?; - let floor: Option = - sqlx::query_scalar("SELECT NULLIF(current_setting('buzz.created_at_floor', true), '')") - .fetch_one(&pool) - .await?; - let isolation: String = sqlx::query_scalar("SHOW transaction_isolation") - .fetch_one(&pool) - .await?; - pool.close().await; - Ok((floor, isolation)) - } - - #[tokio::test] - #[ignore = "requires Postgres"] - async fn writer_hook_modes_distinguish_floor_and_isolation_verdicts() { - let admin = PgPool::connect(&admin_url().await) - .await - .expect("connect admin"); - let base = admin_url().await; - let idx = base.rfind('/').expect("database URL path"); - - let (safe_pool, safe_name) = create_scratch_db(&admin, "hook_modes_safe").await; - safe_pool.close().await; - let safe_url = format!("{}/{}", &base[..idx], safe_name); - let (floor, isolation) = hook_mode_observation(&safe_url, WriterHookMode::Combined) - .await - .expect("combined hook"); - assert_eq!( - floor, - Some(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - ); - assert_eq!(isolation, "read committed"); - let (floor, isolation) = hook_mode_observation(&safe_url, WriterHookMode::Separate) - .await - .expect("separate hooks"); - assert_eq!( - floor, None, - "last after_connect registration must replace floor setup" - ); - assert_eq!(isolation, "read committed"); - let (floor, isolation) = - hook_mode_observation(&safe_url, WriterHookMode::CombinedWithoutIsolation) - .await - .expect("floor-only hook"); - assert_eq!( - floor, - Some(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - ); - assert_eq!(isolation, "read committed"); - let (floor, isolation) = hook_mode_observation(&safe_url, WriterHookMode::None) - .await - .expect("no-hook control"); - assert_eq!(floor, None); - assert_eq!(isolation, "read committed"); - - let (unsafe_pool, unsafe_name) = create_scratch_db(&admin, "hook_modes_unsafe").await; - unsafe_pool.close().await; - sqlx::query(sqlx::AssertSqlSafe(format!( - "ALTER DATABASE {unsafe_name} SET default_transaction_isolation = 'repeatable read'" - ))) - .execute(&admin) - .await - .expect("set unsafe default"); - let unsafe_url = format!("{}/{}", &base[..idx], unsafe_name); - assert!( - hook_mode_observation(&unsafe_url, WriterHookMode::Combined) - .await - .is_err(), - "combined hook must reject unsafe isolation" - ); - assert!( - hook_mode_observation(&unsafe_url, WriterHookMode::Separate) - .await - .is_err(), - "separate last isolation hook rejects but still lacks floor on safe arm" - ); - let (floor, isolation) = - hook_mode_observation(&unsafe_url, WriterHookMode::CombinedWithoutIsolation) - .await - .expect("floor-only mutant admits unsafe default"); - assert_eq!( - floor, - Some(replica_fence::CREATED_AT_FLOOR_SECS.to_string()) - ); - assert_eq!(isolation, "repeatable read"); - let (floor, isolation) = hook_mode_observation(&unsafe_url, WriterHookMode::None) - .await - .expect("no-hook mutant admits unsafe default"); - assert_eq!(floor, None); - assert_eq!(isolation, "repeatable read"); - - for name in [safe_name, unsafe_name] { - sqlx::query(sqlx::AssertSqlSafe(format!( - "DROP DATABASE {name} WITH (FORCE)" - ))) - .execute(&admin) - .await - .expect("drop hook mode database"); - } - } - #[test] fn writer_pool_safety_hook_is_single_and_composed() { let source = include_str!("lib.rs"); diff --git a/crates/buzz-db/src/migration.rs b/crates/buzz-db/src/migration.rs index 2c22a90e6b..069a41a049 100644 --- a/crates/buzz-db/src/migration.rs +++ b/crates/buzz-db/src/migration.rs @@ -496,106 +496,6 @@ mod tests { .collect() } - fn migrations_missing_community_write_fence_attachment() -> Vec { - let mut migrations: Vec<_> = MIGRATOR.iter().collect(); - migrations.sort_by_key(|migration| migration.version); - migrations - .into_iter() - .filter(|migration| migration.version > 29) - .flat_map(|migration| { - let statements = split_sql_statements(migration.sql.as_str()); - let attachments = statements - .iter() - .filter_map(|statement| { - let normalized = normalize_sql(statement); - if !normalized.contains("attach_community_write_fence(") { - return None; - } - let call = normalized.split_once("attach_community_write_fence(")?.1; - let table = call - .split(')') - .next()? - .trim() - .split("::") - .next()? - .trim_matches(|ch| ch == '\'' || ch == '"') - .rsplit('.') - .next()? - .to_owned(); - (!table.is_empty()).then_some(table) - }) - .collect::>(); - statements.into_iter().filter_map(move |statement| { - let normalized = normalize_sql(&statement); - let table = if normalized.starts_with("create table") - && normalized.contains("community_id") - && !normalized.contains(" partition of ") - { - identifier_after_keyword(&statement, "create table") - } else if normalized.starts_with("alter table") - && normalized.contains("add") - && normalized.contains("community_id") - { - identifier_after_keyword(&statement, "alter table") - } else { - None - }?; - (!attachments.contains(&table)).then(|| { - format!( - "migration {} introduces {table}.community_id without attach_community_write_fence('{table}'::regclass)", - migration.version - ) - }) - }) - }) - .collect() - } - - fn community_write_fence_attachment_violations(sql: &str) -> Vec { - let statements = split_sql_statements(sql); - let attachments = statements - .iter() - .filter_map(|statement| { - let normalized = normalize_sql(statement); - if !normalized.contains("attach_community_write_fence(") { - return None; - } - let call = normalized.split_once("attach_community_write_fence(")?.1; - let table = call - .split(')') - .next()? - .trim() - .split("::") - .next()? - .trim_matches(|ch| ch == '\'' || ch == '"') - .rsplit('.') - .next()? - .to_owned(); - (!table.is_empty()).then_some(table) - }) - .collect::>(); - statements - .into_iter() - .filter_map(|statement| { - let normalized = normalize_sql(&statement); - let table = if normalized.starts_with("create table") - && normalized.contains("community_id") - && !normalized.contains(" partition of ") - { - identifier_after_keyword(&statement, "create table") - } else if normalized.starts_with("alter table") - && normalized.contains("add") - && normalized.contains("community_id") - { - identifier_after_keyword(&statement, "alter table") - } else { - None - }?; - (!attachments.contains(&table)).then_some(table) - }) - .collect() - } - fn scoped_constraint_lints(sql: &str, scoped_tables: &BTreeSet) -> Vec { let mut constraints = table_constraints(sql, scoped_tables); constraints.extend(alter_table_constraints(sql, scoped_tables)); @@ -668,7 +568,7 @@ mod tests { let mut migrations: Vec<_> = MIGRATOR.iter().collect(); migrations.sort_by_key(|migration| migration.version); - assert_eq!(migrations.len(), 30); + assert_eq!(migrations.len(), 29); assert_eq!(migrations[0].version, 1); assert_eq!(&*migrations[0].description, "initial schema"); assert!(migrations[0] @@ -1071,55 +971,10 @@ mod tests { assert!(deletion.contains("prevent_community_deletion_request_retargeting")); assert!(deletion.contains("prevent_community_deletion_approval_removal")); - // Bounded deletion retries are additive because the deletion migration - // is checksum-pinned once deployed. - assert_eq!(migrations[29].version, 30); - let bounded_retries = migrations[29].sql.as_str(); - assert!(bounded_retries.contains("ALTER TABLE community_deletion_requests")); - assert!(bounded_retries.contains("ADD COLUMN retry_stage")); - assert!(bounded_retries.contains("'approved', 'fenced', 'drained'")); - assert!(!deletion.contains("retry_stage")); + assert!(deletion.contains("retry_stage TEXT CHECK")); assert!(desired_schema.contains("retry_stage TEXT CHECK")); } - #[test] - fn post_deletion_migrations_attach_every_new_community_write_fence() { - let violations = migrations_missing_community_write_fence_attachment(); - assert!( - violations.is_empty(), - "migrations after 0029 must explicitly attach every new community write fence:\n{}", - violations.join("\n") - ); - } - - #[test] - fn community_write_fence_attachment_lint_covers_create_and_alter() { - let missing = r#" - CREATE TABLE created_late ( - community_id UUID NOT NULL, - id UUID NOT NULL - ); - CREATE TABLE altered_late (id UUID NOT NULL); - ALTER TABLE altered_late ADD COLUMN community_id UUID NOT NULL; - "#; - assert_eq!( - community_write_fence_attachment_violations(missing), - vec!["created_late", "altered_late"] - ); - - let attached = r#" - CREATE TABLE created_late ( - community_id UUID NOT NULL, - id UUID NOT NULL - ); - SELECT attach_community_write_fence('created_late'::regclass); - CREATE TABLE altered_late (id UUID NOT NULL); - ALTER TABLE altered_late ADD COLUMN community_id UUID NOT NULL; - SELECT attach_community_write_fence('altered_late'::regclass); - "#; - assert!(community_write_fence_attachment_violations(attached).is_empty()); - } - #[test] fn migration_lint_detects_tables_missing_community_id_by_default() { let sql = r#" diff --git a/crates/buzz-db/tests/community_fenced_writes.rs b/crates/buzz-db/tests/community_fenced_writes.rs deleted file mode 100644 index ff385ee46f..0000000000 --- a/crates/buzz-db/tests/community_fenced_writes.rs +++ /dev/null @@ -1,1370 +0,0 @@ -//! Repository-complete structural guard for production writes to community-fenced tables. - -use std::collections::{BTreeMap, BTreeSet}; -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::Command; - -use sha2::{Digest, Sha256}; -use sqlparser::ast::{ - visit_statements, BinaryOperator, Expr, FromTable, FunctionArg, FunctionArgExpr, - FunctionArguments, Insert, ObjectName, Query, Select, SelectItem, SetExpr, Statement, - TableFactor, TableObject, Value, -}; -use sqlparser::dialect::PostgreSqlDialect; -use sqlparser::parser::Parser; - -const AUTHORITY: &[&str] = buzz_db::deletion::EXPECTED_SCOPED_TABLES; -const NAMED_SQL: &[(&str, &str, &str)] = &[( - "crates/buzz-db/src/reaction.rs", - "ADD_REACTION_SQL", - "ADD_REACTION_SQL", -)]; - -const CONDITIONAL_SQL: &[(&str, &str, &str)] = &[( - "crates/buzz-db/src/lib.rs", - "replace_parameterized_event", - "statement", -)]; - -#[derive(Clone, Copy)] -struct DynamicSqlInventory { - file: &'static str, - owner: &'static str, - argument: &'static str, - /// SHA-256 of the complete owning function source. - /// This binds a dynamic execution to the SQL-producing dataflow, not merely - /// to a local identifier such as `sql` at the execution call. - owner_fingerprint: &'static str, -} - -const DYNAMIC_SQL: &[DynamicSqlInventory] = &[ - DynamicSqlInventory { - file: "crates/buzz-push-gateway/src/postgres.rs", - owner: "apply_migrations_and_grants", - argument: "AssertSqlSafe(grants)", - owner_fingerprint: "59f2bddd81022c88a9b0367cde36ec53e07c44a09ad643d0db5f3dd83d5244e0", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/channel.rs", - owner: "get_accessible_channels", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "7af8e9f780718222d78e758c31195886532537b7182169b1a72b6f983d8fb5c9", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/channel.rs", - owner: "get_users_bulk", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "ed904badf2c3b9dfeaf6e0c4fe560753732a1b1fcdd6428c8c4f0d27a19c29cb", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/channel.rs", - owner: "update_channel", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "ec7c073f7e618b4d69c79d093b1c8a8320aeb3c8464f550c96a1c0b33162841f", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/deletion.rs", - owner: "inventory_schema", - argument: "AssertSqlSafe(sql)", - owner_fingerprint: "99d744cf8d197e25ee7b620763e7959923517ea7b6f9b5374a0a1c512b5abafb", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/deletion.rs", - owner: "purge_postgres", - argument: "AssertSqlSafe(sql)", - owner_fingerprint: "5d4334785028afef39f9d2f0c42aafce6052a880aeb02d72cd1f102390f5605c", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/deletion.rs", - owner: "verify_postgres_logically_deleted", - argument: "AssertSqlSafe(sql)", - owner_fingerprint: "1faf00caabcd24464fd6d5a83c3006f9e1b4cdc232dfbe9b66c9efbf7b6e7983", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/partition.rs", - owner: "ensure_partition", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "c2802a7c0fec6293a887befe9f78e4becb76b83b1d7315c1e01496bfb5d3b1ff", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/replica_fence.rs", - owner: "reader_supports_aurora_identity", - argument: r#"sqlx::AssertSqlSafe(format!("SELECT {AURORA_IDENTITY_FN}()"))"#, - owner_fingerprint: "3291240355f9b2901a9ec016a4049bccee71dd89fb4b888c8e95573f9519679e", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/replica_fence.rs", - owner: "observe_heartbeat", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "2f7628e054c9e2a195ed92b01994152150d23ea1a4b87b00e30cdaf99b4d829f", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/thread.rs", - owner: "get_thread_replies_on", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "265ee1d19c2b353857518b25185189993e6ee1b729623890c017aeea2dac2721", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/thread.rs", - owner: "get_channel_window_on", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "084c3d9e3cc3621b8602e7d6a653c8af320152a6bac752d95c57e72fb749e4d4", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/usage.rs", - owner: "active_user_counts", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "2fe366035e0eacd5c071e544d81dab8fdcc220e2904b779ec193b95484122571", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/usage.rs", - owner: "active_channel_counts", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "c812cd6eabca88997c6a46933dfcd45a28360716d1aff35086f7ac02cfbdc94a", - }, - DynamicSqlInventory { - file: "crates/buzz-db/src/user.rs", - owner: "update_user_profile", - argument: "sqlx::AssertSqlSafe(sql)", - owner_fingerprint: "2f2be400087028ede8d9281ea5c2b073dbd06f08ef628116f6c3160ddfe3990d", - }, -]; - -// Exact owners whose QueryBuilder classification cannot be proved from the -// extracted literal fragments alone. Their complete owner source is pinned so -// each exception becomes stale on any edit. -struct QueryBuilderException { - file: &'static str, - owner: &'static str, - owner_fingerprint: &'static str, -} - -const QUERY_BUILDER_EXCEPTIONS: &[QueryBuilderException] = &[ - QueryBuilderException { - file: "crates/buzz-db/src/lib.rs", - owner: "insert_mentions", - owner_fingerprint: "c888a340801e900d6511de81c5d63c6f87afa030a87847b9df8b23846904b213", - }, - QueryBuilderException { - file: "crates/buzz-db/src/event.rs", - owner: "query_events_on", - owner_fingerprint: "f06a482fa31252f376ca6cec4a83e9e2675f052810e5d8bbfe6784a7dd44dc45", - }, - QueryBuilderException { - file: "crates/buzz-db/src/event.rs", - owner: "count_events_on", - owner_fingerprint: "7f48326ae5789c421653ed85fc612242e9a7b054ec9915cda13b4d294dd107f2", - }, - QueryBuilderException { - file: "crates/buzz-db/src/event.rs", - owner: "get_last_message_at_bulk", - owner_fingerprint: "a3e9ef0f2eea7df90f052380e85fa7f96cd80ac540969d8ab8319fb135d98312", - }, - QueryBuilderException { - file: "crates/buzz-db/src/event.rs", - owner: "get_events_by_ids_on", - owner_fingerprint: "6f18d34658052e2437767c7ff8063cf3e28dfc11a5b8a2c001c329dc1114a056", - }, - QueryBuilderException { - file: "crates/buzz-db/src/channel.rs", - owner: "get_member_counts_bulk", - owner_fingerprint: "877c8e955a81bd28b8477a4967b67e3036ff95be8e7c3e33a4413119b2cc58aa", - }, - QueryBuilderException { - file: "crates/buzz-search/src/query.rs", - owner: "search", - owner_fingerprint: "ca8adf19af50a6c7310bc2e9d1c7ed7be000677815e9a2fe46af4b158409f189", - }, -]; - -// PostgreSQL syntax intentionally unsupported by sqlparser 0.62. Each entry is -// an exact, reviewed statement fingerprint; source edits fail closed until the -// statement either parses or this inventory is deliberately updated. -const PARSER_EXCEPTIONS: &[(&str, &str, &str)] = &[ - ( - "crates/buzz-db/src/deletion.rs", - "WITH candidate AS", - "WHERE request.id = candidate.id", - ), - ( - "crates/buzz-db/src/push.rs", - "WITH target AS", - "community_write_allowed(community_id)", - ), - ( - "crates/buzz-db/src/push.rs", - "WITH candidates AS", - "WHERE o.community_id = $1", - ), - ( - "crates/buzz-db/src/lib.rs", - "ON CONFLICT (lower(host))", - "communities.deletion_state = 'active'", - ), - ( - "crates/buzz-db/src/lib.rs", - "ON CONFLICT (lower(host)) DO NOTHING", - "RETURNING id, host", - ), -]; - -fn repo_root() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .and_then(Path::parent) - .expect("workspace root") - .to_owned() -} - -fn authority() -> BTreeSet { - AUTHORITY.iter().map(|table| (*table).to_owned()).collect() -} - -fn workspace_crates() -> BTreeSet { - fs::read_dir(repo_root().join("crates")) - .expect("read crates directory") - .filter_map(Result::ok) - .filter(|entry| entry.path().join("Cargo.toml").is_file()) - .map(|entry| entry.file_name().to_string_lossy().into_owned()) - .collect() -} - -fn production_roots() -> Vec { - let roots = workspace_crates() - .into_iter() - .map(|name| repo_root().join("crates").join(name).join("src")) - .filter(|path| path.is_dir()) - .collect::>(); - assert!(!roots.is_empty(), "workspace has no production crate roots"); - roots -} - -fn ast_matches_for_rule(roots: &[PathBuf], rule: &str) -> Vec { - let mut command = Command::new(repo_root().join("bin/ast-grep")); - command - .arg("scan") - .arg("--rule") - .arg(repo_root().join(rule)) - .arg("--json=stream"); - command.args(roots); - let output = command.output().expect("run ast-grep"); - assert!( - output.status.success(), - "ast-grep failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - String::from_utf8(output.stdout) - .expect("ast-grep utf8") - .lines() - .filter(|line| !line.is_empty()) - .map(|line| serde_json::from_str(line).expect("ast-grep json line")) - .collect() -} - -fn ast_matches(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule(roots, "scripts/lints/community_fenced_writes.yml") -} - -fn all_sqlx_matches(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule(roots, "scripts/lints/community_sqlx_calls.yml") -} - -fn querybuilder_mutations(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule(roots, "scripts/lints/community_querybuilder_mutations.yml") -} - -fn querybuilder_builds(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule(roots, "scripts/lints/community_querybuilder_builds.yml") -} - -fn querybuilder_control_flow(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule( - roots, - "scripts/lints/community_querybuilder_control_flow.yml", - ) -} - -fn test_module_ranges(roots: &[PathBuf]) -> BTreeMap> { - let mut cfg_offsets = BTreeMap::>::new(); - for matched in ast_matches_for_rule(roots, "scripts/lints/community_cfg_test_attributes.yml") { - let path = PathBuf::from(matched["file"].as_str().expect("cfg(test) file")); - let offset = matched["range"]["byteOffset"]["start"] - .as_u64() - .expect("cfg offset"); - cfg_offsets.entry(path).or_default().push(offset); - } - - let mut ranges = BTreeMap::>::new(); - for matched in ast_matches_for_rule(roots, "scripts/lints/community_modules.yml") { - let path = PathBuf::from(matched["file"].as_str().expect("module file")); - let start = matched["range"]["byteOffset"]["start"] - .as_u64() - .expect("module start"); - let end = matched["range"]["byteOffset"]["end"] - .as_u64() - .expect("module end"); - let source = fs::read_to_string(&path).expect("read module source"); - let attributed = cfg_offsets.get(&path).is_some_and(|offsets| { - offsets.iter().any(|offset| { - *offset < start - && source[*offset as usize..start as usize] - .trim() - .starts_with("#[cfg(test)]") - }) - }); - if attributed { - ranges.entry(path).or_default().push((start, end)); - } - } - ranges -} - -#[derive(Debug)] -struct FunctionOwner { - path: PathBuf, - start: u64, - end: u64, - name: String, - text: String, -} - -fn function_owners(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule(roots, "scripts/lints/community_function_items.yml") - .into_iter() - .filter_map(|matched| { - let text = matched["text"].as_str()?.to_owned(); - let signature = regex_lite::Regex::new(r"(?:async\s+)?fn\s+([A-Za-z_][A-Za-z0-9_]*)") - .expect("function signature regex"); - let name = signature.captures(&text)?.get(1)?.as_str().to_owned(); - Some(FunctionOwner { - path: PathBuf::from(matched["file"].as_str()?), - start: matched["range"]["byteOffset"]["start"].as_u64()?, - end: matched["range"]["byteOffset"]["end"].as_u64()?, - name, - text, - }) - }) - .collect() -} - -fn const_sql_definitions(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule(roots, "scripts/lints/community_const_sql.yml") -} - -fn conditional_sql_definitions(roots: &[PathBuf]) -> Vec { - ast_matches_for_rule(roots, "scripts/lints/community_conditional_sql.yml") -} - -fn normalize_token(token: &str) -> String { - token - .chars() - .filter(|ch| !ch.is_ascii_whitespace()) - .collect() -} - -fn source_fingerprint(source: &str) -> String { - let normalized = source - .bytes() - .filter(|byte| !byte.is_ascii_whitespace()) - .collect::>(); - hex::encode(Sha256::digest(normalized)) -} - -fn dynamic_inventory_matches( - entry: &DynamicSqlInventory, - relative: &str, - owner: &FunctionOwner, - sql_token: &str, -) -> bool { - relative == entry.file - && owner.name == entry.owner - && normalize_token(sql_token) == normalize_token(entry.argument) - && source_fingerprint(&owner.text) == entry.owner_fingerprint -} - -fn querybuilder_exception_matches( - entry: &QueryBuilderException, - relative: &str, - owner: &FunctionOwner, -) -> bool { - relative == entry.file - && owner.name == entry.owner - && source_fingerprint(&owner.text) == entry.owner_fingerprint -} - -fn rust_string_value(token: &str) -> Option { - if let Ok(literal) = litrs::StringLit::parse(token) { - return Some(literal.value().to_owned()); - } - if token.starts_with('"') && token.ends_with('"') { - let continued = regex_lite::Regex::new( - r"\\ -[ \t]*", - ) - .expect("line-continuation regex") - .replace_all(token, ""); - return serde_json::from_str(&continued).ok(); - } - None -} - -fn basename(name: &ObjectName) -> String { - name.0 - .last() - .and_then(|part| part.as_ident()) - .map(|ident| ident.value.to_ascii_lowercase()) - .unwrap_or_default() -} - -fn table_factor_name(factor: &TableFactor) -> Option { - match factor { - TableFactor::Table { name, .. } => Some(basename(name)), - _ => None, - } -} - -fn target_aliases(factor: &TableFactor, target: &str) -> BTreeSet { - let mut aliases = BTreeSet::from([target.to_owned()]); - if let TableFactor::Table { - alias: Some(alias), .. - } = factor - { - aliases.insert(alias.name.value.to_ascii_lowercase()); - } - aliases -} - -fn is_community_column(expr: &Expr, aliases: &BTreeSet) -> bool { - match expr { - Expr::Identifier(ident) => ident.value.eq_ignore_ascii_case("community_id"), - Expr::CompoundIdentifier(parts) if parts.len() >= 2 => { - parts - .last() - .is_some_and(|part| part.value.eq_ignore_ascii_case("community_id")) - && parts - .get(parts.len() - 2) - .is_some_and(|part| aliases.contains(&part.value.to_ascii_lowercase())) - } - _ => false, - } -} - -fn is_placeholder(expr: &Expr) -> bool { - matches!(expr, Expr::Value(value) if matches!(&value.value, Value::Placeholder(_))) -} - -fn is_allowed_function(expr: &Expr, aliases: &BTreeSet) -> bool { - let Expr::Function(function) = expr else { - return false; - }; - if basename(&function.name) != "community_write_allowed" { - return false; - } - let FunctionArguments::List(arguments) = &function.args else { - return false; - }; - matches!(arguments.args.as_slice(), [FunctionArg::Unnamed(FunctionArgExpr::Expr(argument))] if is_community_column(argument, aliases)) -} - -fn predicate_is_directly_gated(selection: Option<&Expr>, aliases: &BTreeSet) -> bool { - let Some(selection) = selection else { - return false; - }; - match selection { - Expr::Nested(expr) => predicate_is_directly_gated(Some(expr), aliases), - Expr::BinaryOp { - left, - op: BinaryOperator::And, - right, - } => { - predicate_is_directly_gated(Some(left), aliases) - || predicate_is_directly_gated(Some(right), aliases) - } - Expr::BinaryOp { - left, - op: BinaryOperator::Or, - right, - } => { - predicate_is_directly_gated(Some(left), aliases) - && predicate_is_directly_gated(Some(right), aliases) - } - Expr::BinaryOp { - left, - op: BinaryOperator::Eq, - right, - } => { - (is_community_column(left, aliases) && is_placeholder(right)) - || (is_placeholder(left) && is_community_column(right, aliases)) - } - Expr::InSubquery { subquery, .. } | Expr::Subquery(subquery) => { - query_contains_direct_gate(subquery) - } - expr => is_allowed_function(expr, aliases), - } -} - -fn select_aliases(select: &Select) -> BTreeSet { - select - .from - .iter() - .filter_map(|table| { - table_factor_name(&table.relation) - .map(|target| target_aliases(&table.relation, &target)) - }) - .flatten() - .collect() -} - -fn select_projection_expr(select: &Select, index: usize) -> Option<&Expr> { - match select.projection.get(index)? { - SelectItem::UnnamedExpr(expr) - | SelectItem::ExprWithAlias { expr, .. } - | SelectItem::ExprWithAliases { expr, .. } => Some(expr), - SelectItem::QualifiedWildcard(..) | SelectItem::Wildcard(..) => None, - } -} - -fn query_contains_direct_gate(query: &Query) -> bool { - match query.body.as_ref() { - SetExpr::Select(select) => { - predicate_is_directly_gated(select.selection.as_ref(), &select_aliases(select)) - } - SetExpr::Query(query) => query_contains_direct_gate(query), - SetExpr::SetOperation { left, right, .. } => { - set_expr_contains_direct_gate(left) && set_expr_contains_direct_gate(right) - } - _ => false, - } -} - -fn set_expr_contains_direct_gate(set: &SetExpr) -> bool { - match set { - SetExpr::Select(select) => { - predicate_is_directly_gated(select.selection.as_ref(), &select_aliases(select)) - } - SetExpr::Query(query) => query_contains_direct_gate(query), - SetExpr::SetOperation { left, right, .. } => { - set_expr_contains_direct_gate(left) && set_expr_contains_direct_gate(right) - } - _ => false, - } -} - -fn select_projected_community_is_safe(select: &Select, index: usize) -> bool { - let aliases = select_aliases(select); - let Some(projected) = select_projection_expr(select, index) else { - return false; - }; - match projected { - Expr::Value(value) if matches!(&value.value, Value::Placeholder(_)) => true, - expr if is_allowed_function(expr, &aliases) => true, - expr if is_community_column(expr, &aliases) => { - let projected_alias = match expr { - Expr::CompoundIdentifier(parts) if parts.len() >= 2 => { - BTreeSet::from([parts[parts.len() - 2].value.to_ascii_lowercase()]) - } - Expr::Identifier(_) if select.from.len() == 1 => aliases, - _ => return false, - }; - predicate_is_directly_gated(select.selection.as_ref(), &projected_alias) - } - _ => false, - } -} - -fn set_projected_community_is_safe(set: &SetExpr, index: usize) -> bool { - match set { - SetExpr::Select(select) => select_projected_community_is_safe(select, index), - SetExpr::Query(query) => projected_community_is_safe(query, index), - SetExpr::SetOperation { left, right, .. } => { - set_projected_community_is_safe(left, index) - && set_projected_community_is_safe(right, index) - } - SetExpr::Values(values) => values.rows.iter().all(|row| { - row.get(index).is_some_and(|expr| { - matches!(expr, Expr::Value(value) if matches!(&value.value, Value::Placeholder(_))) - }) - }), - _ => false, - } -} - -fn projected_community_is_safe(query: &Query, index: usize) -> bool { - set_projected_community_is_safe(query.body.as_ref(), index) -} - -fn mutation_violation(statement: &Statement, fenced: &BTreeSet) -> Option { - match statement { - Statement::Delete(delete) => { - let tables = match &delete.from { - FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables, - }; - let factor = &tables.first()?.relation; - let target = table_factor_name(factor)?; - if !fenced.contains(&target) { - return None; - } - let aliases = target_aliases(factor, &target); - (!predicate_is_directly_gated(delete.selection.as_ref(), &aliases)).then(|| { - format!("fleet DELETE from {target} lacks a direct tenant predicate or community_write_allowed") - }) - } - Statement::Update(update) => { - let factor = &update.table.relation; - let target = table_factor_name(factor)?; - if !fenced.contains(&target) { - return None; - } - let aliases = target_aliases(factor, &target); - (!predicate_is_directly_gated(update.selection.as_ref(), &aliases)).then(|| { - format!("fleet UPDATE of {target} lacks a direct tenant predicate or community_write_allowed") - }) - } - Statement::Insert(Insert { - table, - columns, - source, - .. - }) => { - let TableObject::TableName(table_name) = table else { - return None; - }; - let target = basename(table_name); - if !fenced.contains(&target) { - return None; - } - if columns.is_empty() { - return Some(format!( - "INSERT into {target} omits its target column list; community_id provenance is unknown" - )); - } - let Some(community_index) = columns - .iter() - .position(|column| basename(column) == "community_id") - else { - return Some(format!( - "INSERT into {target} omits community_id from its target columns" - )); - }; - let Some(source) = source else { return None }; - (!projected_community_is_safe(source, community_index)).then(|| { - format!( - "INSERT into {target} does not prove target community_id is tenant-bound or gated" - ) - }) - } - _ => None, - } -} - -fn statement_is_mutation(statement: &Statement) -> bool { - matches!( - statement, - Statement::Delete(_) | Statement::Update(_) | Statement::Insert(_) - ) -} - -fn mutation_violations( - statements: &Vec, - fenced: &BTreeSet, -) -> (usize, Vec) { - let mut mutation_count = 0; - let mut violations = Vec::new(); - let traversal = visit_statements(statements, |statement| { - if statement_is_mutation(statement) { - mutation_count += 1; - if let Some(violation) = mutation_violation(statement, fenced) { - violations.push(violation); - } - } - std::ops::ControlFlow::<()>::Continue(()) - }); - assert!(matches!(traversal, std::ops::ControlFlow::Continue(()))); - (mutation_count, violations) -} - -fn parsed_querybuilder_is_read_only( - statements: &Vec, - fenced: &BTreeSet, -) -> bool { - !statements.is_empty() - && statements - .iter() - .all(|statement| matches!(statement, Statement::Query(_))) - && mutation_violations(statements, fenced).0 == 0 -} - -fn parse_sql(sql: &str) -> Result, String> { - Parser::parse_sql(&PostgreSqlDialect {}, sql).map_err(|error| error.to_string()) -} - -fn inspect_matches(matches: &[serde_json::Value], roots: &[PathBuf]) -> Vec { - let owners = function_owners(roots); - inspect_matches_with_owners(matches, roots, &owners) -} - -fn inspect_matches_with_owners( - matches: &[serde_json::Value], - roots: &[PathBuf], - owners: &[FunctionOwner], -) -> Vec { - let fenced = authority(); - let root = repo_root(); - let mut violations = Vec::new(); - let mut observed_parser_exceptions = BTreeSet::new(); - let const_defs = const_sql_definitions(roots); - let conditional_defs = conditional_sql_definitions(roots); - let test_ranges = test_module_ranges(roots); - let production_files = roots - .iter() - .flat_map(|source| { - walkdir::WalkDir::new(source) - .into_iter() - .filter_map(Result::ok) - }) - .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "rs")) - .map(|entry| entry.into_path()) - .collect::>(); - assert!(!production_files.is_empty()); - - for matched in matches { - let path = PathBuf::from(matched["file"].as_str().expect("match file")); - assert!( - production_files.contains(&path), - "match escaped production roots: {path:?}" - ); - let offset = matched["range"]["byteOffset"]["start"] - .as_u64() - .expect("match offset"); - if test_ranges.get(&path).is_some_and(|ranges| { - ranges - .iter() - .any(|(start, end)| *start <= offset && offset < *end) - }) { - continue; - } - let line = matched["range"]["start"]["line"].as_u64().unwrap_or(0) + 1; - let sql_token = matched["metaVariables"]["single"] - .get("SQL") - .and_then(|sql| sql["text"].as_str()); - if sql_token.is_none() { - // QueryBuilder execution is validated against its constructor in - // `querybuilder_executions_have_one_owned_creation` below. - continue; - } - let sql_token = sql_token.expect("checked SQL metavariable"); - let Some(sql) = rust_string_value(sql_token) else { - let relative = path.strip_prefix(&root).unwrap_or(&path).to_string_lossy(); - let owner = owners - .iter() - .find(|owner| owner.path == path && owner.start <= offset && offset < owner.end); - let dynamic_matches = owner.map_or(0, |owner| { - DYNAMIC_SQL - .iter() - .filter(|entry| dynamic_inventory_matches(entry, &relative, owner, sql_token)) - .count() - }); - let named_matches = NAMED_SQL - .iter() - .filter(|(file, symbol, expression)| { - relative == *file - && sql_token == *expression - && const_defs.iter().any(|definition| { - Path::new(definition["file"].as_str().unwrap_or_default()) == path - && definition["metaVariables"]["single"]["NAME"]["text"].as_str() - == Some(*symbol) - }) - }) - .count(); - let conditional_matches = owner.map_or(0, |owner| { - CONDITIONAL_SQL - .iter() - .filter(|(file, expected_owner, expression)| { - relative == *file - && owner.name == *expected_owner - && sql_token == *expression - && conditional_defs.iter().any(|definition| { - Path::new(definition["file"].as_str().unwrap_or_default()) == path - && definition["range"]["byteOffset"]["start"] - .as_u64() - .is_some_and(|start| { - owner.start <= start && start < owner.end - }) - && definition["metaVariables"]["single"]["NAME"]["text"] - .as_str() - == Some(*expression) - }) - }) - .count() - }); - let total = dynamic_matches + named_matches + conditional_matches; - if total != 1 { - violations.push(format!( - "{relative}:{line}: nonliteral SQL expression must match exactly one inventory entry; got {total}: {sql_token}" - )); - } - continue; - }; - match parse_sql(&sql) { - Ok(statements) => { - for statement in statements { - if let Some(message) = mutation_violation(&statement, &fenced) { - violations.push(format!( - "{}:{line}: {message}", - path.strip_prefix(&root).unwrap_or(&path).display() - )); - } - } - } - Err(error) - if sql.to_ascii_lowercase().contains("insert into") - || sql.to_ascii_lowercase().contains("update ") - || sql.to_ascii_lowercase().contains("delete from") => - { - let relative = path.strip_prefix(&root).unwrap_or(&path).to_string_lossy(); - let excepted = PARSER_EXCEPTIONS.iter().any(|(file, start, guard)| { - relative == *file && sql.contains(start) && sql.contains(guard) - }); - if excepted { - observed_parser_exceptions.insert(relative.into_owned()); - } else { - violations.push(format!( - "{relative}:{line}: production write SQL did not parse: {error}" - )); - } - } - Err(_) => {} - } - } - - for entry in DYNAMIC_SQL { - let path = root.join(entry.file); - let count = matches - .iter() - .filter(|matched| { - Path::new(matched["file"].as_str().unwrap_or_default()) == path - && matched["metaVariables"]["single"] - .get("SQL") - .and_then(|sql| sql["text"].as_str()) - .is_some_and(|token| { - owners.iter().any(|owner| { - matched["range"]["byteOffset"]["start"] - .as_u64() - .is_some_and(|offset| { - owner.path == path - && owner.start <= offset - && offset < owner.end - && dynamic_inventory_matches( - entry, entry.file, owner, token, - ) - }) - }) - }) - }) - .count(); - if count != 1 { - violations.push(format!( - "{}: dynamic SQL inventory must match exactly once: owner={:?} argument={:?} owner_fingerprint={:?}; got {count}", - entry.file, entry.owner, entry.argument, entry.owner_fingerprint - )); - } - } - for (relative, symbol, expression) in NAMED_SQL { - let path = root.join(relative); - let definitions = const_defs - .iter() - .filter(|definition| { - Path::new(definition["file"].as_str().unwrap_or_default()) == path - && definition["metaVariables"]["single"]["NAME"]["text"].as_str() - == Some(*symbol) - }) - .count(); - let executions = matches - .iter() - .filter(|matched| { - Path::new(matched["file"].as_str().unwrap_or_default()) == path - && matched["metaVariables"]["single"] - .get("SQL") - .and_then(|sql| sql["text"].as_str()) - == Some(*expression) - }) - .count(); - if definitions != 1 || executions == 0 { - violations.push(format!("{relative}: named SQL inventory mismatch for {symbol}: definitions={definitions} executions={executions}")); - } - } - for (relative, expected_owner, expression) in CONDITIONAL_SQL { - let path = root.join(relative); - let definitions = conditional_defs - .iter() - .filter(|definition| { - Path::new(definition["file"].as_str().unwrap_or_default()) == path - && definition["metaVariables"]["single"]["NAME"]["text"].as_str() - == Some(*expression) - && definition["range"]["byteOffset"]["start"] - .as_u64() - .is_some_and(|offset| { - owners.iter().any(|owner| { - owner.path == path - && owner.name == *expected_owner - && owner.start <= offset - && offset < owner.end - }) - }) - }) - .count(); - if definitions != 1 { - violations.push(format!("{relative}: conditional SQL inventory mismatch: owner={expected_owner} expression={expression} definitions={definitions}")); - } - } - violations -} - -fn querybuilder_violations(roots: &[PathBuf]) -> Vec { - let production_scan = roots - .iter() - .any(|root| root.starts_with(repo_root().join("crates")) && root.ends_with("src")); - let owners = function_owners(roots); - let mutations = querybuilder_mutations(roots); - let builds = querybuilder_builds(roots); - let control_flow = querybuilder_control_flow(roots); - let mut found = Vec::new(); - let mut observed_querybuilder_exceptions = BTreeSet::new(); - for build in &builds { - let path = PathBuf::from(build["file"].as_str().expect("builder file")); - let offset = build["range"]["byteOffset"]["start"] - .as_u64() - .expect("builder offset"); - let owner = owners - .iter() - .find(|owner| owner.path == path && owner.start <= offset && offset < owner.end); - let Some(owner) = owner else { - continue; - }; - if !owner.text.contains("QueryBuilder") { - continue; - } - let receiver = build["metaVariables"]["single"]["QB"]["text"] - .as_str() - .expect("builder receiver"); - if !receiver - .chars() - .all(|ch| ch == '_' || ch.is_ascii_alphanumeric()) - { - found.push(format!( - "{}: QueryBuilder build has non-identifier receiver {receiver:?}", - path.display() - )); - continue; - } - let mut all_owned = mutations - .iter() - .filter(|mutation| { - Path::new(mutation["file"].as_str().unwrap_or_default()) == path - && mutation["range"]["byteOffset"]["start"] - .as_u64() - .is_some_and(|start| owner.start <= start && start < owner.end) - && mutation["metaVariables"]["single"] - .get("QB") - .and_then(|value| value["text"].as_str()) - == Some(receiver) - }) - .collect::>(); - all_owned.sort_by_key(|mutation| { - mutation["range"]["byteOffset"]["start"] - .as_u64() - .unwrap_or_default() - }); - let constructors = all_owned - .iter() - .filter(|mutation| { - mutation["text"] - .as_str() - .is_some_and(|text| text.starts_with("let mut ")) - }) - .count(); - if constructors != 1 { - let relative = path - .strip_prefix(repo_root()) - .unwrap_or(&path) - .to_string_lossy(); - let excepted = QUERY_BUILDER_EXCEPTIONS - .iter() - .find(|entry| querybuilder_exception_matches(entry, &relative, owner)); - if let Some(entry) = excepted { - observed_querybuilder_exceptions.insert((entry.file, entry.owner)); - } else { - found.push(format!("{}: QueryBuilder {receiver} must have exactly one direct constructor in owner {}; got {constructors}", path.display(), owner.name)); - } - continue; - } - if owner.text.contains(&format!("let mut alias = {receiver}")) - || owner.text.contains(&format!("let alias = {receiver}")) - || owner.text.contains(&format!("{receiver} =")) && constructors > 1 - { - found.push(format!( - "{}: QueryBuilder {receiver} has ambiguous alias/reassignment", - path.display() - )); - continue; - } - let constructor_offset = all_owned - .iter() - .find(|mutation| { - mutation["text"] - .as_str() - .is_some_and(|text| text.starts_with("let mut ")) - }) - .and_then(|mutation| mutation["range"]["byteOffset"]["start"].as_u64()) - .expect("exactly one QueryBuilder constructor"); - if constructor_offset >= offset { - found.push(format!( - "{}: QueryBuilder {receiver} constructor does not precede build", - path.display() - )); - continue; - } - let fragments = all_owned - .iter() - .filter(|mutation| { - mutation["range"]["byteOffset"]["start"] - .as_u64() - .is_some_and(|start| constructor_offset <= start && start < offset) - }) - .filter_map(|mutation| mutation["metaVariables"]["single"].get("SQL")?["text"].as_str()) - .map(|token| rust_string_value(token).ok_or_else(|| token.to_owned())) - .collect::, _>>(); - let fragments = match fragments { - Ok(fragments) => fragments, - Err(token) => { - found.push(format!( - "{}: QueryBuilder {receiver} has nonliteral fragment {token}", - path.display() - )); - continue; - } - }; - // QueryBuilder appends string fragments exactly, so parse the same - // byte sequence that will be executed. The AST, not raw keyword - // spelling, decides whether conditional state is safe to ignore. - let reconstructed = fragments.concat(); - let relative = path - .strip_prefix(repo_root()) - .unwrap_or(&path) - .to_string_lossy(); - let inventory_entry = QUERY_BUILDER_EXCEPTIONS - .iter() - .find(|entry| querybuilder_exception_matches(entry, &relative, owner)); - if let Some(entry) = inventory_entry { - observed_querybuilder_exceptions.insert((entry.file, entry.owner)); - } - let inventoried = inventory_entry.is_some(); - let parsed = parse_sql(&reconstructed); - let fenced = authority(); - let is_read_only = parsed - .as_ref() - .is_ok_and(|statements| parsed_querybuilder_is_read_only(statements, &fenced)); - // Fingerprinted exceptions cover builders whose dynamic bind helpers - // prevent a complete literal reconstruction, not parsed mutations. - if is_read_only || (inventoried && parsed.is_err()) { - continue; - } - // Parsed reads and fingerprinted, unparseable exact-owner exceptions - // are safe to ignore here. Every other shape must pass the conditional- - // state gate and AST mutation validation below. - let owner_control_flow = control_flow.iter().filter(|control| { - Path::new(control["file"].as_str().unwrap_or_default()) == path - && control["range"]["byteOffset"]["start"] - .as_u64() - .zip(control["range"]["byteOffset"]["end"].as_u64()) - .is_some_and(|(start, end)| owner.start <= start && end <= owner.end) - }); - let mut state_offsets = all_owned - .iter() - .filter_map(|mutation| { - mutation["range"]["byteOffset"]["start"] - .as_u64() - .filter(|mutation_offset| { - constructor_offset <= *mutation_offset && *mutation_offset < offset - }) - }) - .collect::>(); - state_offsets.push(offset); - let build_state_is_conditional = state_offsets.into_iter().any(|state_offset| { - owner_control_flow.clone().any(|control| { - control["range"]["byteOffset"]["start"] - .as_u64() - .zip(control["range"]["byteOffset"]["end"].as_u64()) - .is_some_and(|(start, end)| start <= state_offset && state_offset < end) - }) - }); - if build_state_is_conditional { - found.push(format!( - "{}: mutating QueryBuilder {receiver} constructor, mutation, or build is control-flow-dependent", - path.display() - )); - continue; - } - let statements = match parsed { - Ok(statements) if mutation_violations(&statements, &fenced).0 > 0 => statements, - Ok(statements) => { - found.push(format!( - "{}: QueryBuilder {receiver} is neither recursively read-only nor mutation-bearing and has no exact inventory: statements={statements:?}; fragments={fragments:?}", - path.display() - )); - continue; - } - Err(error) => { - found.push(format!("{}: QueryBuilder {receiver} failed reconstruction and has no exact inventory: {error}; fragments={fragments:?}", path.display())); - continue; - } - }; - found.extend( - mutation_violations(&statements, &fenced) - .1 - .into_iter() - .map(|violation| { - format!( - "{}: mutating QueryBuilder {receiver}: {violation}", - path.display() - ) - }), - ); - } - if production_scan { - for entry in QUERY_BUILDER_EXCEPTIONS { - if !observed_querybuilder_exceptions.contains(&(entry.file, entry.owner)) { - found.push(format!( - "{}: QueryBuilder inventory entry for {} is stale or unobserved", - entry.file, entry.owner - )); - } - } - } - found -} - -#[test] -fn scanner_rejects_dynamic_producer_changes_with_the_same_argument_token() { - const REVIEWED_PRODUCER: &str = r#""UPDATE channels SET {}, updated_at = NOW() WHERE community_id = ${param_idx} AND id = ${channel_param_idx} AND deleted_at IS NULL""#; - const UNSAFE_PRODUCER: &str = r#""DELETE FROM relay_invites WHERE expires_at < now() /* {} ${param_idx} ${channel_param_idx} */""#; - - let roots = production_roots(); - let matches = ast_matches(&roots); - let mut owners = function_owners(&roots); - let entry = DYNAMIC_SQL - .iter() - .find(|entry| { - entry.file == "crates/buzz-db/src/channel.rs" && entry.owner == "update_channel" - }) - .expect("update_channel dynamic inventory"); - let owner_index = owners - .iter() - .position(|owner| owner.path == repo_root().join(entry.file) && owner.name == entry.owner) - .expect("update_channel owner"); - let source = owners[owner_index].text.clone(); - assert!( - source.contains(REVIEWED_PRODUCER), - "reviewed update_channel SQL producer drifted" - ); - assert!( - source.contains(entry.argument), - "calibration requires the inventoried argument token" - ); - - for (description, mutant) in [ - ( - "unsafe SQL producer", - source.replacen(REVIEWED_PRODUCER, UNSAFE_PRODUCER, 1), - ), - ( - "additional dynamic execution in the reviewed owner", - source.replacen( - entry.argument, - &format!( - "{}); sqlx::query(sqlx::AssertSqlSafe(attacker_sql", - entry.argument - ), - 1, - ), - ), - ] { - assert!( - mutant.contains(entry.argument), - "{description} mutant changed the inventoried SQL argument token" - ); - owners[owner_index].text = mutant; - let violations = inspect_matches_with_owners(&matches, &roots, &owners); - assert!( - violations.iter().any(|violation| { - violation.contains("crates/buzz-db/src/channel.rs") - && violation.contains( - "nonliteral SQL expression must match exactly one inventory entry", - ) - && violation.contains(entry.argument) - }), - "{description} mutant escaped: {violations:?}" - ); - } -} - -#[test] -fn every_workspace_sql_execution_is_scanned_and_querybuilders_are_owned() { - let roots = production_roots(); - let matches = all_sqlx_matches(&roots); - assert!( - !matches.is_empty(), - "SQL execution coverage probe matched no calls" - ); - for matched in &matches { - assert!( - matched["file"].as_str().is_some(), - "SQL execution match has no source file" - ); - } - let violations = querybuilder_violations(&roots); - assert!( - violations.is_empty(), - "QueryBuilder violations: -{}", - violations.join( - " -" - ) - ); -} - -#[test] -fn production_fenced_writers_are_structurally_tenant_bound_or_gated() { - let roots = production_roots(); - let matches = ast_matches(&roots); - let violations = inspect_matches(&matches, &roots); - assert!( - violations.is_empty(), - "community fenced-writer violations:\n{}", - violations.join("\n") - ); -} - -#[test] -fn scanner_rejects_calibrated_bad_shapes_through_the_real_extractor() { - let fixture = repo_root().join("crates/buzz-db/tests/fixtures/community_fenced_writes"); - let matches = ast_matches(std::slice::from_ref(&fixture)); - let violations = inspect_matches(&matches, std::slice::from_ref(&fixture)); - assert!( - violations.iter().any(|v| v.contains("bad_insert_select")), - "fleet INSERT SELECT control escaped: {violations:?}" - ); - assert!( - violations - .iter() - .any(|v| v.contains("bad_unrelated_subquery")), - "unrelated-subquery DELETE control escaped: {violations:?}" - ); - for (fixture_name, description) in [ - ("bad_or_delete", "OR-broadened DELETE"), - ( - "bad_insert_unfenced_source", - "unfenced-source INSERT SELECT", - ), - ( - "bad_insert_unrelated_alias", - "unrelated-alias INSERT SELECT", - ), - ( - "bad_dynamic_assert_sql_safe", - "unreviewed AssertSqlSafe dynamic SQL", - ), - ( - "bad_insert_implicit_columns", - "implicit-column INSERT SELECT", - ), - ("bad_generic_query", "generic query constructor"), - ("bad_query_with", "query_with constructor"), - ("bad_query_as_with", "query_as_with constructor"), - ("bad_query_scalar_with", "query_scalar_with constructor"), - ("bad_raw_sql", "raw_sql constructor"), - ( - "bad_dynamic_second_in_owner", - "second dynamic expression in inventoried owner", - ), - ] { - assert!( - violations - .iter() - .any(|violation| violation.contains(fixture_name)), - "{description} control escaped: {violations:?}" - ); - } - let builder_violations = querybuilder_violations(std::slice::from_ref(&fixture)); - for fixture_name in [ - "bad_querybuilder_direct", - "bad_querybuilder_split", - "bad_querybuilder_two_receivers", - "bad_querybuilder_branch", - "bad_querybuilder_alias", - "bad_querybuilder_conditional_push", - "bad_querybuilder_conditional_whitespace", - "bad_querybuilder_unconditional_cte", - "bad_querybuilder_post_build_push", - ] { - assert!( - builder_violations - .iter() - .any(|violation| violation.contains(fixture_name)), - "QueryBuilder control {fixture_name} escaped: {builder_violations:?}" - ); - } - assert!( - builder_violations.iter().any(|violation| { - violation.contains("bad_querybuilder_conditional_push") - && violation.contains("control-flow-dependent") - }), - "comment-separated conditional DELETE QueryBuilder tenant predicate escaped the control-flow gate: {builder_violations:?}" - ); - assert!( - builder_violations - .iter() - .filter(|violation| { - violation.contains("bad_querybuilder_conditional_whitespace") - && violation.contains("control-flow-dependent") - }) - .count() - == 5, - "tab/newline direct DML or conditional DELETE/UPDATE/INSERT CTE QueryBuilder escaped the control-flow gate: {builder_violations:?}" - ); - assert!( - builder_violations - .iter() - .filter(|violation| { - violation.contains("bad_querybuilder_unconditional_cte") - && (violation.contains("fleet DELETE") - || violation.contains("fleet UPDATE") - || violation.contains("omits community_id")) - }) - .count() - == 4, - "unconditional or nested DELETE/UPDATE/INSERT CTE QueryBuilder escaped recursive tenant validation: {builder_violations:?}" - ); - assert!( - !builder_violations - .iter() - .any(|violation| violation.contains("good_querybuilder_conditional_select")), - "conditional SELECT QueryBuilder was rejected: {builder_violations:?}" - ); - assert!( - !violations.iter().any(|v| v.contains("good_tenant_delete")), - "tenant-bound control was rejected: {violations:?}" - ); - assert!( - !violations.iter().any(|v| v.contains("good_gated_delete")), - "gated control was rejected: {violations:?}" - ); - assert!( - !violations - .iter() - .any(|v| v.contains("good_cfg_test_module")), - "actual cfg(test) module was treated as production: {violations:?}" - ); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_assert_sql_safe.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_assert_sql_safe.rs deleted file mode 100644 index 16b27bce29..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_assert_sql_safe.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn bad_dynamic_assert_sql_safe(dynamic_sql: String) { - sqlx::query(sqlx::AssertSqlSafe(dynamic_sql)); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_second_in_owner.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_second_in_owner.rs deleted file mode 100644 index dcb9bdeb8e..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_dynamic_second_in_owner.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn update_channel() { - let sql = "UPDATE channels SET name=$1 WHERE community_id=$2".to_string(); - sqlx::query(sqlx::AssertSqlSafe(sql)); - let attacker_sql = "DELETE FROM relay_invites WHERE expires_at < now()".to_string(); - sqlx::query::(sqlx::AssertSqlSafe(attacker_sql)); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_generic_query.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_generic_query.rs deleted file mode 100644 index ce4ab38735..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_generic_query.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn mutant(dynamic_sql: String) { - sqlx::query::(sqlx::AssertSqlSafe(dynamic_sql)); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_implicit_columns.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_implicit_columns.rs deleted file mode 100644 index 3d8c85bdba..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_implicit_columns.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn bad_insert_implicit_columns() { - sqlx::query("INSERT INTO relay_invites SELECT community_id, id, token_hash, 'member', NULL, 0, now(), 'x', now() FROM relay_invites"); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_select.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_select.rs deleted file mode 100644 index e46a8931ae..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_select.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn bad_insert_select() { - sqlx::query("INSERT INTO relay_invites (community_id, token_hash, expires_at, created_by) SELECT community_id, token_hash, now(), 'x' FROM relay_invites"); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unfenced_source.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unfenced_source.rs deleted file mode 100644 index 917fdc5002..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unfenced_source.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn bad_insert_unfenced_source() { - sqlx::query("INSERT INTO relay_invites (community_id, token_hash, expires_at, created_by) SELECT id, decode(repeat('00', 32), 'hex'), now(), 'x' FROM communities"); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unrelated_alias.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unrelated_alias.rs deleted file mode 100644 index ba6b35e9b4..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_insert_unrelated_alias.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn bad_insert_unrelated_alias() { - sqlx::query("INSERT INTO relay_invites (community_id, token_hash, expires_at, created_by) SELECT ri.community_id, ri.token_hash, now(), 'x' FROM relay_invites ri, event_tags et WHERE et.community_id = $1"); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_or_delete.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_or_delete.rs deleted file mode 100644 index b80442d764..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_or_delete.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn bad_or_delete() { - sqlx::query("DELETE FROM relay_invites WHERE community_id = $1 OR expires_at < now()"); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_production_mod_tests.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_production_mod_tests.rs deleted file mode 100644 index 9273d49fe5..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_production_mod_tests.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod tests { - fn production_writer_despite_name() { - sqlx::query("DELETE FROM relay_invites WHERE expires_at < now()"); - } -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_as_with.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_as_with.rs deleted file mode 100644 index 8d6c4af26d..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_as_with.rs +++ /dev/null @@ -1 +0,0 @@ -fn mutant(sql: String, args: sqlx::postgres::PgArguments) { sqlx::query_as_with::(sqlx::AssertSqlSafe(sql), args); } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_scalar_with.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_scalar_with.rs deleted file mode 100644 index 9d02c49135..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_scalar_with.rs +++ /dev/null @@ -1 +0,0 @@ -fn mutant(sql: String, args: sqlx::postgres::PgArguments) { sqlx::query_scalar_with::(sqlx::AssertSqlSafe(sql), args); } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_with.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_with.rs deleted file mode 100644 index b0abb5a5f8..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_query_with.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn mutant(dynamic_sql: String, args: sqlx::postgres::PgArguments) { - sqlx::query_with(sqlx::AssertSqlSafe(dynamic_sql), args); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_alias.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_alias.rs deleted file mode 100644 index b0d5e502d0..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_alias.rs +++ /dev/null @@ -1,5 +0,0 @@ -fn mutant() { - let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at = if flag { - sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE community_id=$1") - } else { - sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at = - sqlx::QueryBuilder::new("DELETE/**/FROM relay_invites"); - if scope_to_tenant { - qb.push(" WHERE community_id = $1"); - } - let _ = qb.build(); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs deleted file mode 100644 index dc6d7af1c2..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_conditional_whitespace.rs +++ /dev/null @@ -1,50 +0,0 @@ -fn tab_separated_update(scope_to_tenant: bool) { - let mut qb: sqlx::QueryBuilder = - sqlx::QueryBuilder::new("UPDATE\trelay_invites SET expires_at = now()"); - if scope_to_tenant { - qb.push(" WHERE community_id = $1"); - } - let _ = qb.build(); -} - -fn newline_separated_insert(scope_to_tenant: bool) { - let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( - "INSERT\nINTO relay_invites (community_id, token_hash) VALUES ($1, $2)", - ); - if scope_to_tenant { - qb.push(" ON CONFLICT DO NOTHING"); - } - let _ = qb.build(); -} - -fn conditional_delete_cte(scope_to_tenant: bool) { - let mut qb: sqlx::QueryBuilder = - sqlx::QueryBuilder::new("WITH gone AS (DELETE FROM relay_invites"); - if scope_to_tenant { - qb.push(" WHERE community_id = $1"); - } - qb.push(" RETURNING 1) SELECT 1"); - let _ = qb.build(); -} - -fn conditional_update_cte(scope_to_tenant: bool) { - let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( - "WITH changed AS (UPDATE relay_invites SET expires_at = now()", - ); - if scope_to_tenant { - qb.push(" WHERE community_id = $1"); - } - qb.push(" RETURNING 1) SELECT 1"); - let _ = qb.build(); -} - -fn conditional_insert_cte(scope_to_tenant: bool) { - let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( - "WITH made AS (INSERT INTO relay_invites (community_id, token_hash)", - ); - if scope_to_tenant { - qb.push(" VALUES ($1, $2)"); - } - qb.push(" RETURNING 1) SELECT 1"); - let _ = qb.build(); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_direct.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_direct.rs deleted file mode 100644 index 4d257aaac4..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_direct.rs +++ /dev/null @@ -1 +0,0 @@ -fn mutant() { let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at < now()"); let _ = qb.build(); } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_post_build_push.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_post_build_push.rs deleted file mode 100644 index 0a24219046..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_post_build_push.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn mutant() { - let mut qb: sqlx::QueryBuilder = - sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at < now()"); - let _ = qb.build(); - qb.push(" AND community_id = $1"); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_split.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_split.rs deleted file mode 100644 index bdb3e195bb..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_split.rs +++ /dev/null @@ -1 +0,0 @@ -fn mutant() { let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE "); qb.push("expires_at < now()"); let _ = qb.build(); } diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_two_receivers.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_two_receivers.rs deleted file mode 100644 index 47c9ccc08b..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_querybuilder_two_receivers.rs +++ /dev/null @@ -1,6 +0,0 @@ -fn mutant() { - let mut safe: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE community_id=$1"); - let mut unsafe_q: sqlx::QueryBuilder = sqlx::QueryBuilder::new("DELETE FROM relay_invites WHERE expires_at = sqlx::QueryBuilder::new( - "WITH gone AS (DELETE FROM relay_invites RETURNING 1) SELECT 1", - ); - let _ = qb.build(); -} - -fn unconditional_update_cte() { - let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( - "WITH changed AS (UPDATE relay_invites SET expires_at = now() RETURNING 1) SELECT 1", - ); - let _ = qb.build(); -} - -fn unconditional_insert_cte() { - let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( - "WITH made AS (INSERT INTO relay_invites (token_hash) VALUES ($1) RETURNING 1) SELECT 1", - ); - let _ = qb.build(); -} - -fn nested_unconditional_delete_cte() { - let mut qb: sqlx::QueryBuilder = sqlx::QueryBuilder::new( - "WITH outer_cte AS (WITH gone AS (DELETE FROM relay_invites RETURNING 1) SELECT 1) SELECT 1", - ); - let _ = qb.build(); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_raw_sql.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_raw_sql.rs deleted file mode 100644 index 5e866cdb81..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_raw_sql.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn mutant(dynamic_sql: String) { - sqlx::raw_sql(sqlx::AssertSqlSafe(dynamic_sql)); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_unrelated_subquery.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_unrelated_subquery.rs deleted file mode 100644 index ad7b80f2c7..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/bad_unrelated_subquery.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn bad_unrelated_subquery() { - sqlx::query("DELETE FROM relay_invites WHERE expires_at < now() AND EXISTS (SELECT 1 FROM communities WHERE id=$1)"); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_cfg_test_module.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_cfg_test_module.rs deleted file mode 100644 index ba8fda895c..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_cfg_test_module.rs +++ /dev/null @@ -1,6 +0,0 @@ -#[cfg(test)] -mod postgres_tests { - fn ignored_test_writer() { - sqlx::query("DELETE FROM relay_invites WHERE expires_at < now()"); - } -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_gated_delete.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_gated_delete.rs deleted file mode 100644 index 5ef73d84b0..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_gated_delete.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn good_gated_delete() { - sqlx::query("DELETE FROM relay_invites WHERE expires_at < now() AND community_write_allowed(community_id)"); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_querybuilder_conditional_select.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_querybuilder_conditional_select.rs deleted file mode 100644 index d164955f73..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_querybuilder_conditional_select.rs +++ /dev/null @@ -1,8 +0,0 @@ -fn control(filter_by_kind: bool) { - let mut qb: sqlx::QueryBuilder = - sqlx::QueryBuilder::new("SELECT id FROM events WHERE community_id = $1"); - if filter_by_kind { - qb.push(" AND kind = $2"); - } - let _ = qb.build(); -} diff --git a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_tenant_delete.rs b/crates/buzz-db/tests/fixtures/community_fenced_writes/good_tenant_delete.rs deleted file mode 100644 index 2d578d5bd0..0000000000 --- a/crates/buzz-db/tests/fixtures/community_fenced_writes/good_tenant_delete.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn good_tenant_delete() { - sqlx::query("DELETE FROM relay_invites WHERE community_id=$1 AND expires_at < $2"); -} diff --git a/migrations/0029_community_deletion.sql b/migrations/0029_community_deletion.sql index 60737351c9..a140b35681 100644 --- a/migrations/0029_community_deletion.sql +++ b/migrations/0029_community_deletion.sql @@ -40,6 +40,10 @@ CREATE TABLE community_deletion_requests ( lease_until TIMESTAMPTZ, attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + retry_stage TEXT CHECK (retry_stage IS NULL OR retry_stage IN ( + 'approved', 'fenced', 'drained', 'bindings_removed', + 'postgres_purged', 'cache_purged', 'logically_verified' + )), next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_error TEXT, last_error_at TIMESTAMPTZ, diff --git a/migrations/0030_bound_deletion_retries.sql b/migrations/0030_bound_deletion_retries.sql deleted file mode 100644 index 88906536f5..0000000000 --- a/migrations/0030_bound_deletion_retries.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Restore the per-stage retry streak needed to bound transient deletion --- failures. Migration 0029 is already deployed and checksum-pinned, so this --- schema addition must remain additive. -ALTER TABLE community_deletion_requests - ADD COLUMN retry_stage TEXT CHECK (retry_stage IS NULL OR retry_stage IN ( - 'approved', 'fenced', 'drained', 'bindings_removed', - 'postgres_purged', 'cache_purged', 'logically_verified' - )); diff --git a/scripts/lints/community_cfg_test_attributes.yml b/scripts/lints/community_cfg_test_attributes.yml deleted file mode 100644 index a956a79461..0000000000 --- a/scripts/lints/community_cfg_test_attributes.yml +++ /dev/null @@ -1,7 +0,0 @@ -id: community-cfg-test-attribute -message: Actual cfg(test) attribute used to identify test-only syntax -severity: hint -language: rust -rule: - kind: attribute_item - regex: 'cfg\(test\)' diff --git a/scripts/lints/community_conditional_sql.yml b/scripts/lints/community_conditional_sql.yml deleted file mode 100644 index b29b9dac0d..0000000000 --- a/scripts/lints/community_conditional_sql.yml +++ /dev/null @@ -1,6 +0,0 @@ -id: community-conditional-sql -message: Conditional SQL local definition -severity: hint -language: rust -rule: - pattern: 'let $NAME = if $COND { $A } else { $B };' diff --git a/scripts/lints/community_const_sql.yml b/scripts/lints/community_const_sql.yml deleted file mode 100644 index 0061cfd576..0000000000 --- a/scripts/lints/community_const_sql.yml +++ /dev/null @@ -1,6 +0,0 @@ -id: community-const-sql -message: Named SQL constant definition -severity: hint -language: rust -rule: - pattern: 'const $NAME: $TYPE = $VALUE;' diff --git a/scripts/lints/community_fenced_writes.yml b/scripts/lints/community_fenced_writes.yml deleted file mode 100644 index bc3223d12a..0000000000 --- a/scripts/lints/community_fenced_writes.yml +++ /dev/null @@ -1,16 +0,0 @@ -id: community-sql-execution -message: SQL execution constructor/builder used to prove repository-complete source coverage -severity: hint -language: rust -rule: - any: - - pattern: sqlx::query($SQL) - - pattern: sqlx::query::<$$$T>($SQL) - - pattern: sqlx::query_with($SQL, $ARGS) - - pattern: sqlx::query_as($SQL) - - pattern: sqlx::query_as::<$$$T>($SQL) - - pattern: sqlx::query_as_with::<$$$T>($SQL, $ARGS) - - pattern: sqlx::query_scalar($SQL) - - pattern: sqlx::query_scalar::<$$$T>($SQL) - - pattern: sqlx::query_scalar_with::<$$$T>($SQL, $ARGS) - - pattern: sqlx::raw_sql($SQL) diff --git a/scripts/lints/community_function_items.yml b/scripts/lints/community_function_items.yml deleted file mode 100644 index 2b0a3b0a22..0000000000 --- a/scripts/lints/community_function_items.yml +++ /dev/null @@ -1,6 +0,0 @@ -id: community-function-item -message: Rust function item used to bind dynamic SQL to its exact owner -severity: hint -language: rust -rule: - kind: function_item diff --git a/scripts/lints/community_modules.yml b/scripts/lints/community_modules.yml deleted file mode 100644 index 26f095bbbe..0000000000 --- a/scripts/lints/community_modules.yml +++ /dev/null @@ -1,6 +0,0 @@ -id: community-module -message: Rust module range used to bind cfg(test) attributes to syntax -severity: hint -language: rust -rule: - kind: mod_item diff --git a/scripts/lints/community_querybuilder_builds.yml b/scripts/lints/community_querybuilder_builds.yml deleted file mode 100644 index 6187da1d44..0000000000 --- a/scripts/lints/community_querybuilder_builds.yml +++ /dev/null @@ -1,9 +0,0 @@ -id: community-querybuilder-build -message: QueryBuilder execution call bound to its exact receiver symbol -severity: hint -language: rust -rule: - any: - - pattern: $QB.build() - - pattern: $QB.build_query_as::<$$$T>() - - pattern: $QB.build_query_scalar::<$$$T>() diff --git a/scripts/lints/community_querybuilder_control_flow.yml b/scripts/lints/community_querybuilder_control_flow.yml deleted file mode 100644 index 7397cfbda7..0000000000 --- a/scripts/lints/community_querybuilder_control_flow.yml +++ /dev/null @@ -1,13 +0,0 @@ -id: community-querybuilder-control-flow -message: Control-flow region used to reject ambiguous QueryBuilder state -severity: hint -language: rust -rule: - any: - - kind: if_expression - - kind: for_expression - - kind: while_expression - - kind: loop_expression - - kind: match_expression - - kind: closure_expression - - kind: async_block diff --git a/scripts/lints/community_querybuilder_creations.yml b/scripts/lints/community_querybuilder_creations.yml deleted file mode 100644 index 020ee5f6da..0000000000 --- a/scripts/lints/community_querybuilder_creations.yml +++ /dev/null @@ -1,10 +0,0 @@ -id: community-querybuilder-creation -message: QueryBuilder construction bound to its owning function -severity: hint -language: rust -rule: - any: - - pattern: QueryBuilder::new($SQL) - - pattern: QueryBuilder::<$$$T>::new($SQL) - - pattern: sqlx::QueryBuilder::new($SQL) - - pattern: sqlx::QueryBuilder::<$$$T>::new($SQL) diff --git a/scripts/lints/community_querybuilder_mutations.yml b/scripts/lints/community_querybuilder_mutations.yml deleted file mode 100644 index 1d8fa7777f..0000000000 --- a/scripts/lints/community_querybuilder_mutations.yml +++ /dev/null @@ -1,11 +0,0 @@ -id: community-querybuilder-mutation -message: QueryBuilder constructor/push fragments bound to one receiver -severity: hint -language: rust -rule: - any: - - pattern: 'let mut $QB: $TYPE = QueryBuilder::new($SQL);' - - pattern: 'let mut $QB = QueryBuilder::new($SQL);' - - pattern: 'let mut $QB: $TYPE = sqlx::QueryBuilder::new($SQL);' - - pattern: 'let mut $QB = sqlx::QueryBuilder::new($SQL);' - - pattern: '$QB.push($SQL)' diff --git a/scripts/lints/community_sqlx_calls.yml b/scripts/lints/community_sqlx_calls.yml deleted file mode 100644 index bc3223d12a..0000000000 --- a/scripts/lints/community_sqlx_calls.yml +++ /dev/null @@ -1,16 +0,0 @@ -id: community-sql-execution -message: SQL execution constructor/builder used to prove repository-complete source coverage -severity: hint -language: rust -rule: - any: - - pattern: sqlx::query($SQL) - - pattern: sqlx::query::<$$$T>($SQL) - - pattern: sqlx::query_with($SQL, $ARGS) - - pattern: sqlx::query_as($SQL) - - pattern: sqlx::query_as::<$$$T>($SQL) - - pattern: sqlx::query_as_with::<$$$T>($SQL, $ARGS) - - pattern: sqlx::query_scalar($SQL) - - pattern: sqlx::query_scalar::<$$$T>($SQL) - - pattern: sqlx::query_scalar_with::<$$$T>($SQL, $ARGS) - - pattern: sqlx::raw_sql($SQL) From f559e69e93590a1f794e455cb8016a89d744c789 Mon Sep 17 00:00:00 2001 From: cid Date: Thu, 6 Aug 2026 11:48:27 -0700 Subject: [PATCH 51/51] fix: send policy close on community deletion Co-authored-by: cid Signed-off-by: cid --- crates/buzz-relay/src/audio/handler.rs | 105 +++++++++++++-- crates/buzz-relay/src/connection.rs | 170 ++++++++++++++++++++++--- crates/buzz-relay/src/state.rs | 131 +++++++++++++++---- 3 files changed, 355 insertions(+), 51 deletions(-) diff --git a/crates/buzz-relay/src/audio/handler.rs b/crates/buzz-relay/src/audio/handler.rs index 16cd56209c..4c158eab0c 100644 --- a/crates/buzz-relay/src/audio/handler.rs +++ b/crates/buzz-relay/src/audio/handler.rs @@ -25,7 +25,7 @@ use bytes::Bytes; use futures_util::{SinkExt, StreamExt}; use nostr::{EventBuilder, Kind, Tag}; use serde::Deserialize; -use tokio::sync::{mpsc, OwnedSemaphorePermit, Semaphore}; +use tokio::sync::{mpsc, watch, OwnedSemaphorePermit, Semaphore}; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; use uuid::Uuid; @@ -38,7 +38,7 @@ use buzz_core::StoredEvent; use buzz_pubsub::EventTopic; use crate::audio::room::PeerCtrl; -use crate::state::{run_registered_community_connection, AppState}; +use crate::state::{run_registered_community_connection, AppState, CommunityConnectionControl}; /// Maximum binary frame size: 4 KB is generous for a single Opus packet. const MAX_AUDIO_FRAME_BYTES: usize = 4096; @@ -149,6 +149,7 @@ async fn handle_audio_connection( _permit: OwnedSemaphorePermit, ) { let cancel = CancellationToken::new(); + let control = CommunityConnectionControl::new(cancel); let community_id = tenant.community(); let registry = Arc::clone(&state.community_connections); let check_state = Arc::clone(&state); @@ -157,9 +158,11 @@ async fn handle_audio_connection( ®istry, Uuid::new_v4(), community_id, - cancel.clone(), + control, move || async move { check_state.db.is_community_active(community_id).await }, - move || handle_active_audio_connection(socket, run_state, tenant, channel_id, cancel), + move |control| { + handle_active_audio_connection(socket, run_state, tenant, channel_id, control) + }, ) .await; } @@ -169,8 +172,10 @@ async fn handle_active_audio_connection( state: Arc, tenant: TenantContext, channel_id: Uuid, - cancel: CancellationToken, + control: CommunityConnectionControl, ) { + let cancel = control.cancellation_token(); + let disconnect_reason = control.disconnect_reason(); let (mut ws_send, mut ws_recv) = socket.split(); let challenge = generate_challenge(); @@ -660,7 +665,13 @@ async fn handle_active_audio_connection( let (ctrl_tx, ctrl_rx) = mpsc::channel::(8); let send_cancel = cancel.child_token(); - let send_task = tokio::spawn(send_loop(ws_send, data_rx, ctrl_rx, send_cancel)); + let send_task = tokio::spawn(send_loop( + ws_send, + data_rx, + ctrl_rx, + send_cancel, + disconnect_reason, + )); let hb_cancel = cancel.clone(); let hb_missed = Arc::clone(&missed_pongs); @@ -1056,12 +1067,15 @@ async fn recv_loop( /// /// Control frames (Ping, Pong, Close, control JSON) are drained first on every /// iteration, so heartbeat pings are never starved by audio backpressure. -async fn send_loop( - mut ws_send: futures_util::stream::SplitSink, +async fn send_loop( + mut ws_send: S, mut data_rx: mpsc::Receiver, mut ctrl_rx: mpsc::Receiver, cancel: CancellationToken, -) { + disconnect_reason: watch::Receiver>, +) where + S: futures_util::Sink + Unpin, +{ loop { // Priority: drain all pending control frames before data. while let Ok(ctrl_msg) = ctrl_rx.try_recv() { @@ -1073,7 +1087,10 @@ async fn send_loop( tokio::select! { biased; _ = cancel.cancelled() => { - let _ = ws_send.send(WsMessage::Close(None)).await; + let close = disconnect_reason + .borrow() + .map_or(WsMessage::Close(None), |reason| reason.close_message()); + let _ = ws_send.send(close).await; break; } Some(ctrl_msg) = ctrl_rx.recv() => { @@ -1416,6 +1433,74 @@ mod tests { received } + #[tokio::test] + async fn audio_send_loop_sends_policy_close_when_community_is_deleted() { + use futures_util::Sink; + + struct MockSink { + messages: Arc>>, + } + + impl Sink for MockSink { + type Error = std::io::Error; + + fn poll_ready( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn start_send( + self: std::pin::Pin<&mut Self>, + item: WsMessage, + ) -> Result<(), Self::Error> { + self.messages.lock().expect("mock sink poisoned").push(item); + Ok(()) + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn poll_close( + self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + self.poll_flush(cx) + } + } + + let (_data_tx, data_rx) = mpsc::channel(1); + let (_ctrl_tx, ctrl_rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + let control = CommunityConnectionControl::new(cancel.clone()); + let disconnect_reason = control.disconnect_reason(); + let registry = crate::state::CommunityConnectionRegistry::new(); + let community = buzz_core::CommunityId::from_uuid(Uuid::new_v4()); + let _guard = registry.register(Uuid::new_v4(), community, control); + assert_eq!(registry.disconnect_community(community), 1); + let messages = Arc::new(Mutex::new(Vec::new())); + let sink = MockSink { + messages: Arc::clone(&messages), + }; + + send_loop(sink, data_rx, ctrl_rx, cancel, disconnect_reason).await; + + let messages = messages.lock().expect("mock sink poisoned"); + assert_eq!(messages.len(), 1); + match &messages[0] { + WsMessage::Close(Some(close)) => { + assert_eq!(close.code, axum::extract::ws::close_code::POLICY); + assert_eq!(close.reason.as_str(), "community deleted"); + } + other => panic!("expected one 1008 deletion close, got {other:?}"), + } + } + #[tokio::test] async fn audio_websocket_parser_rejects_oversized_messages_before_handler_reads_them() { assert!( diff --git a/crates/buzz-relay/src/connection.rs b/crates/buzz-relay/src/connection.rs index 72a7eb9126..c37421e7e8 100644 --- a/crates/buzz-relay/src/connection.rs +++ b/crates/buzz-relay/src/connection.rs @@ -8,7 +8,7 @@ use std::time::Duration; use axum::extract::ws::{Message as WsMessage, WebSocket}; use futures_util::{Sink, SinkExt, StreamExt}; -use tokio::sync::{mpsc, Mutex, RwLock}; +use tokio::sync::{mpsc, watch, Mutex, RwLock}; use tokio_util::sync::CancellationToken; use tracing::Instrument as _; use tracing::{debug, info, trace, warn}; @@ -20,7 +20,10 @@ use nostr::Filter; use crate::handlers; use crate::protocol::{ClientMessage, RelayMessage}; -use crate::state::{run_registered_community_connection, AppState}; +use crate::state::{ + run_registered_community_connection, AppState, CommunityConnectionControl, + CommunityDisconnectReason, +}; use buzz_pubsub::EventTopic; /// Maximum time a new socket may hold a connection slot without completing NIP-42 auth. @@ -128,6 +131,7 @@ pub async fn handle_connection( ) { let conn_id = Uuid::new_v4(); let cancel = CancellationToken::new(); + let control = CommunityConnectionControl::new(cancel); let community_id = tenant.community(); let registry = Arc::clone(&state.community_connections); let check_state = Arc::clone(&state); @@ -136,9 +140,9 @@ pub async fn handle_connection( ®istry, conn_id, community_id, - cancel.clone(), + control, move || async move { check_state.db.is_community_active(community_id).await }, - move || handle_active_connection(socket, run_state, addr, tenant, conn_id, cancel), + move |control| handle_active_connection(socket, run_state, addr, tenant, conn_id, control), ) .await; } @@ -149,8 +153,10 @@ async fn handle_active_connection( addr: SocketAddr, tenant: TenantContext, conn_id: Uuid, - cancel: CancellationToken, + control: CommunityConnectionControl, ) { + let cancel = control.cancellation_token(); + let disconnect_reason = control.disconnect_reason(); let permit = match state.conn_semaphore.clone().try_acquire_owned() { Ok(p) => p, Err(_) => { @@ -226,7 +232,14 @@ async fn handle_active_connection( let (ws_send, ws_recv) = socket.split(); let send_cancel = cancel.child_token(); - let send_task = tokio::spawn(send_loop(ws_send, rx, ctrl_rx, restart_rx, send_cancel)); + let send_task = tokio::spawn(send_loop( + ws_send, + rx, + ctrl_rx, + restart_rx, + send_cancel, + disconnect_reason, + )); let missed_pongs = Arc::new(AtomicU8::new(0)); let heartbeat_cancel = cancel.clone(); @@ -310,8 +323,17 @@ async fn send_loop( ctrl_rx: mpsc::Receiver, restart_rx: mpsc::Receiver, cancel: CancellationToken, + disconnect_reason: watch::Receiver>, ) { - send_loop_inner(ws_send, data_rx, ctrl_rx, restart_rx, cancel).await; + send_loop_inner( + ws_send, + data_rx, + ctrl_rx, + restart_rx, + cancel, + disconnect_reason, + ) + .await; } async fn send_loop_inner( @@ -320,6 +342,7 @@ async fn send_loop_inner( mut ctrl_rx: mpsc::Receiver, mut restart_rx: mpsc::Receiver, cancel: CancellationToken, + disconnect_reason: watch::Receiver>, ) where S: Sink + Unpin, { @@ -359,7 +382,10 @@ async fn send_loop_inner( break; } } - let _ = ws_send.send(WsMessage::Close(None)).await; + let close = disconnect_reason + .borrow() + .map_or(WsMessage::Close(None), |reason| reason.close_message()); + let _ = ws_send.send(close).await; break; } Some(ctrl_msg) = ctrl_rx.recv() => { @@ -787,6 +813,17 @@ mod tests { } } + fn ordinary_disconnect_reason() -> watch::Receiver> { + let (_tx, rx) = watch::channel(None); + rx + } + + fn deleted_community_disconnect_reason() -> watch::Receiver> { + let (tx, rx) = watch::channel(None); + tx.send_replace(Some(CommunityDisconnectReason::CommunityDeleted)); + rx + } + fn text_payloads(messages: &[WsMessage]) -> Vec { messages .iter() @@ -823,7 +860,15 @@ mod tests { let (sink, state) = MockSink::new(Some(1)); let (_restart_tx, restart_rx) = mpsc::channel(1); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; let state = state.lock().expect("mock sink poisoned"); assert_eq!(state.flush_count, 1); @@ -844,7 +889,15 @@ mod tests { let (sink, state) = MockSink::new(Some(1)); let (_restart_tx, restart_rx) = mpsc::channel(1); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; let state = state.lock().expect("mock sink poisoned"); assert_eq!(state.flush_count, 1); @@ -870,7 +923,15 @@ mod tests { let (sink, state) = MockSink::new(Some(2)); let (_restart_tx, restart_rx) = mpsc::channel(1); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; let state = state.lock().expect("mock sink poisoned"); assert_eq!(state.flush_count, 2); @@ -894,7 +955,15 @@ mod tests { .expect("queue restart close"); let (sink, state) = MockSink::new(None); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; assert_eq!(flushed_rx.await, Ok(true)); let state = state.lock().expect("mock sink poisoned"); @@ -923,7 +992,15 @@ mod tests { .expect("queue restart close"); let (sink, state) = MockSink::new(Some(1)); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, CancellationToken::new()).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + CancellationToken::new(), + ordinary_disconnect_reason(), + ) + .await; assert_eq!(flushed_rx.await, Ok(false)); let state = state.lock().expect("mock sink poisoned"); @@ -931,6 +1008,59 @@ mod tests { assert_eq!(state.messages.len(), 1, "no fallback close is appended"); } + #[tokio::test] + async fn send_loop_sends_policy_close_when_community_is_deleted() { + let (_data_tx, data_rx) = mpsc::channel(1); + let (_ctrl_tx, ctrl_rx) = mpsc::channel(1); + let (_restart_tx, restart_rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + cancel.cancel(); + + let (sink, state) = MockSink::new(None); + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + cancel, + deleted_community_disconnect_reason(), + ) + .await; + + let state = state.lock().expect("mock sink poisoned"); + assert_eq!(state.messages.len(), 1); + match &state.messages[0] { + WsMessage::Close(Some(close)) => { + assert_eq!(close.code, axum::extract::ws::close_code::POLICY); + assert_eq!(close.reason.as_str(), "community deleted"); + } + other => panic!("expected one 1008 deletion close, got {other:?}"), + } + } + + #[tokio::test] + async fn send_loop_sends_bare_close_for_ordinary_cancellation() { + let (_data_tx, data_rx) = mpsc::channel(1); + let (_ctrl_tx, ctrl_rx) = mpsc::channel(1); + let (_restart_tx, restart_rx) = mpsc::channel(1); + let cancel = CancellationToken::new(); + cancel.cancel(); + + let (sink, state) = MockSink::new(None); + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + cancel, + ordinary_disconnect_reason(), + ) + .await; + + let state = state.lock().expect("mock sink poisoned"); + assert_eq!(state.messages.as_slice(), [WsMessage::Close(None)]); + } + #[tokio::test] async fn send_loop_flushes_queued_control_before_close_on_cancel() { // A ban disconnect queues its `OK false "blocked: …"` reason frame on @@ -951,7 +1081,15 @@ mod tests { let (sink, state) = MockSink::new(None); let (_restart_tx, restart_rx) = mpsc::channel(1); - send_loop_inner(sink, data_rx, ctrl_rx, restart_rx, cancel).await; + send_loop_inner( + sink, + data_rx, + ctrl_rx, + restart_rx, + cancel, + ordinary_disconnect_reason(), + ) + .await; let state = state.lock().expect("mock sink poisoned"); assert_eq!( @@ -966,8 +1104,8 @@ mod tests { other => panic!("expected the ban reason frame first, got {other:?}"), } assert!( - matches!(state.messages[1], WsMessage::Close(_)), - "Close is sent only after the reason frame is flushed" + matches!(state.messages[1], WsMessage::Close(None)), + "ordinary cancellation retains the bare Close after the reason frame" ); } } diff --git a/crates/buzz-relay/src/state.rs b/crates/buzz-relay/src/state.rs index 14a50df7b7..2f544e188c 100644 --- a/crates/buzz-relay/src/state.rs +++ b/crates/buzz-relay/src/state.rs @@ -10,8 +10,7 @@ use axum::body::Bytes; use axum::extract::ws::{Message as WsMessage, Utf8Bytes as WsUtf8Bytes}; use dashmap::DashMap; use futures_util::future::join_all; -use tokio::sync::mpsc; -use tokio::sync::Semaphore; +use tokio::sync::{mpsc, watch, Semaphore}; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; use uuid::Uuid; @@ -37,6 +36,55 @@ use crate::subscription::SubscriptionRegistry; pub(crate) type ScopedPubkeyKey = (CommunityId, [u8; 32]); +/// Why a community-bound socket is being asked to stop. +/// +/// Only deletion is externally attributed today. Ordinary lifecycle exits keep +/// using cancellation alone and therefore retain the existing bare-close +/// behavior. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum CommunityDisconnectReason { + CommunityDeleted, +} + +impl CommunityDisconnectReason { + pub(crate) fn close_message(self) -> WsMessage { + match self { + Self::CommunityDeleted => WsMessage::Close(Some(axum::extract::ws::CloseFrame { + code: axum::extract::ws::close_code::POLICY, + reason: WsUtf8Bytes::from_static("community deleted"), + })), + } + } +} + +/// Per-socket lifecycle controls shared by the registry and the writer. +#[derive(Clone)] +pub(crate) struct CommunityConnectionControl { + cancel: CancellationToken, + reason_tx: watch::Sender>, +} + +impl CommunityConnectionControl { + pub(crate) fn new(cancel: CancellationToken) -> Self { + let (reason_tx, _reason_rx) = watch::channel(None); + Self { cancel, reason_tx } + } + + pub(crate) fn cancellation_token(&self) -> CancellationToken { + self.cancel.clone() + } + + pub(crate) fn disconnect_reason(&self) -> watch::Receiver> { + self.reason_tx.subscribe() + } + + fn disconnect_community(&self) { + self.reason_tx + .send_replace(Some(CommunityDisconnectReason::CommunityDeleted)); + self.cancel.cancel(); + } +} + /// Leaves headroom under the process-wide drain deadline for a stalled writer. const RESTART_CLOSE_ACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); type SlidingWindowCounter = (u32, Instant); @@ -68,7 +116,7 @@ struct ConnEntry { /// registration cancels the token; archival before registration is observed by /// the revalidation. The returned guard removes the entry on every exit path. pub struct CommunityConnectionRegistry { - connections: Arc>, + connections: Arc>, } impl Default for CommunityConnectionRegistry { @@ -86,26 +134,27 @@ impl CommunityConnectionRegistry { } /// Registers one socket and returns a guard that deregisters it on drop. - pub fn register( + pub(crate) fn register( &self, connection_id: Uuid, community_id: CommunityId, - cancel: CancellationToken, + control: CommunityConnectionControl, ) -> CommunityConnectionGuard { self.connections - .insert(connection_id, (community_id, cancel)); + .insert(connection_id, (community_id, control)); CommunityConnectionGuard { connection_id, connections: Arc::clone(&self.connections), } } - /// Cancels every socket type currently bound to `community_id`. + /// Disconnects every socket type currently bound to `community_id` and + /// attributes the close to community deletion. pub fn disconnect_community(&self, community_id: CommunityId) -> usize { let mut closed = 0; for entry in self.connections.iter() { if entry.value().0 == community_id { - entry.value().1.cancel(); + entry.value().1.disconnect_community(); closed += 1; } } @@ -124,7 +173,7 @@ impl CommunityConnectionRegistry { /// Removes a socket lifecycle registration on every handler exit path. pub struct CommunityConnectionGuard { connection_id: Uuid, - connections: Arc>, + connections: Arc>, } impl Drop for CommunityConnectionGuard { @@ -137,20 +186,21 @@ impl Drop for CommunityConnectionGuard { /// /// The ordering is the archival admission invariant: archive-before-query is /// observed by the query, while archive-after-registration sees the token. -pub async fn run_registered_community_connection( +pub(crate) async fn run_registered_community_connection( registry: &CommunityConnectionRegistry, connection_id: Uuid, community_id: CommunityId, - cancel: CancellationToken, + control: CommunityConnectionControl, check_active: Check, run: Run, ) where Check: FnOnce() -> CheckFuture, CheckFuture: Future>, - Run: FnOnce() -> RunFuture, + Run: FnOnce(CommunityConnectionControl) -> RunFuture, RunFuture: Future, { - let _guard = registry.register(connection_id, community_id, cancel.clone()); + let cancel = control.cancel.clone(); + let _guard = registry.register(connection_id, community_id, control.clone()); if !matches!(check_active().await, Ok(true)) { cancel.cancel(); return; @@ -158,7 +208,7 @@ pub async fn run_registered_community_connection