From 4ca0632def89042a3227c858645dfa84616874c1 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:09:18 +0000 Subject: [PATCH 01/15] fix(streams): disabling a stream closes its shards; enabling creates a new stream On UpdateTable StreamEnabled=false the stream is closed: DescribeStream reports DISABLED with the stream's own view type, every shard carries an EndingSequenceNumber (its last record's sequence number, or its starting number when it holds none), DescribeTable omits StreamSpecification, and LatestStreamArn keeps naming the closed stream. Writes record only to open shards. UpdateTable StreamEnabled=true afterwards creates a new stream with a later label and four new shard ids; the closed stream stays describable and listed. ListStreams lists every stream of a table, newest first. Enabling a table whose stream is enabled is refused with "Table already has an enabled stream: TableName: ", and disabling a table with no enabled stream with "Table has no stream to disable: TableName: ". GetShardIterator on a stream ARN with a shard of the table's other stream answers "Requested resource not found: Shard does not exist". A stream change travels alone: an UpdateTable carrying StreamSpecification beside BillingMode or ProvisionedThroughput is refused with "You cannot modify stream status while updating table IOPS", beside any GlobalSecondaryIndexUpdates entry with "You cannot create or delete index while changing stream status", beside DeletionProtectionEnabled with "DeletionProtection modification must be the only operation in the request", and nothing is applied. A disable beside a StreamViewType is refused ("If stream is being disabled, then UpdateViewType must not be specified"). CreateTable with a stream enabled and no view type is refused ("Table is being created with a stream enabled, UpdateViewType is required"); a StreamSpecification without StreamEnabled on CreateTable or UpdateTable gets the model's null-member sentence; a table created with StreamSpecification {StreamEnabled: false} stores nothing of it. ListStreams carries LastEvaluatedStreamArn whenever Limit cut the page, refuses an ExclusiveStartStreamArn of another table ("ExclusiveStartStreamArn provided is invalid: "), Limit under 1 and a start ARN under 37 characters. GetShardIterator refuses a ShardId under 28 or over 65 characters and a SequenceNumber under 21 with the service's constraint sentences, answers AT_SEQUENCE_NUMBER without a number with "SequenceNumber cannot be null", and an unknown stream ARN with "Requested resource not found: Stream does not exist". DescribeStream refuses a short ExclusiveStartShardId and answers an unknown stream of an existing table with "Stream: not found". Shard ids written before v0.1.14 (shardId-{table_name}-{index}) are the table's shards: that stream stays describable, listed and readable after the upgrade, and keeps its view type when disabled (the disable stores {StreamEnabled: false} with the view type and responses omit the member). A disable closes only the stream it decided on. On MongoDB and Cassandra the catalog write of a stream change is conditioned on the specification and label it read, so of two overlapping changes exactly one applies and the other is refused; an enable re-reads the row after inserting its shards and closes them when a disable got in between, so a DISABLED stream has every shard closed. A write racing a disable lands before the ending number on PostgreSQL (the open shards are read FOR SHARE and locked FOR UPDATE by the close) and in no stream on Cassandra (a new stream starts at a clock value drawn at enable time); MongoDB keeps a one-transaction window. Each stream's identity is carried in its shard ids (shardId-{table_id}-{label_millis}{view_type}{index}), so no backend needs a schema change; ids written before this scheme belong to the table's current stream. The rules shared by the four backends live in extenddb_storage::stream_lineage. On Cassandra, whose write paths compute a record's partition without reading the shard rows, a stream's shard is a range of its index's partition. Re-cut on main after 877ef394 (resolving the shard against its stream inside the GetRecords read). The not-found sentences use the one set of helpers in extenddb_storage::util; GetShardIterator with a shard of the table's other stream answers "Requested resource not found: Shard does not exist" (util::SHARD_DOES_NOT_EXIST, measured 2026-10-24), replacing the "Stream: not found." sentence main kept there. The Cassandra get_stream_records keeps main's ownership check (the shard row's table id and util::shard_in_stream) and its two round trips, and reads the records from the shard's physical partition in parallel with the shard row, applying the shard's sequence range to the rows it gets back. The engine keeps main's constraint helper beside the shard id length constants. Closes #482 --- .github/workflows/integration.yml | 2 +- crates/core/src/types/table.rs | 12 + crates/core/src/validation/mod.rs | 33 +- crates/engine/src/backup.rs | 7 +- crates/engine/src/create_table.rs | 1 + crates/engine/src/lib.rs | 18 + crates/engine/src/streams.rs | 77 ++- crates/engine/src/update_table.rs | 232 +++---- crates/engine/src/update_table_tests.rs | 215 +++++++ crates/storage-cassandra/src/create_table.rs | 54 +- crates/storage-cassandra/src/lib.rs | 1 + crates/storage-cassandra/src/stream_engine.rs | 466 ++++---------- .../storage-cassandra/src/stream_lifecycle.rs | 512 ++++++++++++++++ crates/storage-cassandra/src/update_table.rs | 94 ++- .../tests/direct/stream_disable.rs | 389 ++++++++++++ .../storage-cassandra/tests/direct/streams.rs | 10 +- .../tests/direct_integration.rs | 2 + crates/storage-mongodb/src/data_engine.rs | 7 +- crates/storage-mongodb/src/lib.rs | 1 + crates/storage-mongodb/src/stream_engine.rs | 366 +++-------- .../storage-mongodb/src/stream_lifecycle.rs | 312 ++++++++++ .../storage-mongodb/src/stream_live_tests.rs | 270 ++++++++ .../src/stream_records_live_tests.rs | 34 +- crates/storage-mongodb/src/table_engine.rs | 115 ++-- crates/storage-postgres/src/create_table.rs | 29 +- .../storage-postgres/src/data/tx_helpers.rs | 12 +- crates/storage-postgres/src/lib.rs | 1 + crates/storage-postgres/src/stream_engine.rs | 317 ++-------- .../storage-postgres/src/stream_lifecycle.rs | 381 ++++++++++++ crates/storage-postgres/src/update_table.rs | 86 +-- .../storage-postgres/tests/stream_disable.rs | 544 +++++++++++++++++ crates/storage-sqlite/src/create_table.rs | 20 +- crates/storage-sqlite/src/data/tx_helpers.rs | 17 +- crates/storage-sqlite/src/lib.rs | 1 + crates/storage-sqlite/src/stream.rs | 279 ++------- crates/storage-sqlite/src/stream_lifecycle.rs | 243 ++++++++ .../src/stream_lifecycle_tests.rs | 357 +++++++++++ crates/storage-sqlite/src/update_table.rs | 41 +- crates/storage/src/lib.rs | 1 + crates/storage/src/stream_lineage.rs | 578 ++++++++++++++++++ crates/storage/src/stream_lineage_tests.rs | 465 ++++++++++++++ crates/storage/src/util/arn.rs | 26 +- crates/storage/src/util/mod.rs | 6 +- docs/design/07-component-streams.md | 14 +- docs/differences-from-dynamodb.md | 1 + tests/python/test_stream_disable.py | 416 +++++++++++++ tests/python/test_stream_disable_fold.py | 329 ++++++++++ .../python/test_stream_disable_validation.py | 272 +++++++++ 48 files changed, 6066 insertions(+), 1600 deletions(-) create mode 100644 crates/engine/src/update_table_tests.rs create mode 100644 crates/storage-cassandra/src/stream_lifecycle.rs create mode 100644 crates/storage-cassandra/tests/direct/stream_disable.rs create mode 100644 crates/storage-mongodb/src/stream_lifecycle.rs create mode 100644 crates/storage-mongodb/src/stream_live_tests.rs create mode 100644 crates/storage-postgres/src/stream_lifecycle.rs create mode 100644 crates/storage-postgres/tests/stream_disable.rs create mode 100644 crates/storage-sqlite/src/stream_lifecycle.rs create mode 100644 crates/storage-sqlite/src/stream_lifecycle_tests.rs create mode 100644 crates/storage/src/stream_lineage.rs create mode 100644 crates/storage/src/stream_lineage_tests.rs create mode 100644 tests/python/test_stream_disable.py create mode 100644 tests/python/test_stream_disable_fold.py create mode 100644 tests/python/test_stream_disable_validation.py diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index b1de36c0b..c7cb2c458 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -323,7 +323,7 @@ jobs: run: >- cargo test --release -p extenddb-storage-postgres --test vector_control_plane --test put_create_race --test twi_conflict - --test stream_shard_retention --test stream_shard_ownership --test stream_records_page_order --test backup_restore + --test stream_shard_retention --test stream_shard_ownership --test stream_records_page_order --test stream_disable --test backup_restore --test gsi_online_build --test gsi_stale_writer --test table_snapshot # The daemonized server logs to syslog; dump it so server-side failures diff --git a/crates/core/src/types/table.rs b/crates/core/src/types/table.rs index fdebcab30..fa6817ff6 100755 --- a/crates/core/src/types/table.rs +++ b/crates/core/src/types/table.rs @@ -815,8 +815,20 @@ impl TableDescription { /// table is ACTIVE (measured on DynamoDB, and still absent 30 s later). /// Backends report the summary whenever the table has a registered /// restore. + /// - A disabled `StreamSpecification` is omitted: the service carries the + /// member only while a stream is enabled (measured 2026-10-24 on the + /// disable response and on `DescribeTable` afterwards). Backends keep + /// the disabled specification stored, with the closed stream's view + /// type, so `DescribeStream` of that stream can still report it. pub fn prepare_for_response(&mut self) { self.populate_table_throughput_mode_summary(); + if self + .stream_specification + .as_ref() + .is_some_and(|s| !s.stream_enabled) + { + self.stream_specification = None; + } if self.table_status != TableStatus::Creating { self.restore_summary = None; } else if let Some(summary) = self.restore_summary.as_mut() { diff --git a/crates/core/src/validation/mod.rs b/crates/core/src/validation/mod.rs index 3decb4666..0284403a4 100755 --- a/crates/core/src/validation/mod.rs +++ b/crates/core/src/validation/mod.rs @@ -659,15 +659,21 @@ fn validate_index_projections(input: &CreateTableInput) -> Result<(), DynamoDbEr /// # Errors /// /// Returns `DynamoDbError::ValidationException` when `StreamEnabled` is false -/// but a `StreamViewType` is present. +/// but a `StreamViewType` is present, or `StreamEnabled` is true without one +/// (measured 2026-10-24; its order against the other CreateTable checks is not +/// measured, so it sits with the other stream check). fn validate_stream_specification(input: &CreateTableInput) -> Result<(), DynamoDbError> { - if let Some(stream) = &input.stream_specification - && !stream.stream_enabled - && stream.stream_view_type.is_some() - { - return Err(DynamoDbError::ValidationException( - "One or more parameter values were invalid: Table is being created with a stream disabled, UpdateViewType should not be specified".to_owned(), - )); + if let Some(stream) = &input.stream_specification { + if !stream.stream_enabled && stream.stream_view_type.is_some() { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: Table is being created with a stream disabled, UpdateViewType should not be specified".to_owned(), + )); + } + if stream.stream_enabled && stream.stream_view_type.is_none() { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: Table is being created with a stream enabled, UpdateViewType is required".to_owned(), + )); + } } Ok(()) } @@ -3722,6 +3728,17 @@ mod tests { stream_view_type: Some(StreamViewType::NewImage), }); assert!(validate_create_table(&input, &LimitsConfig::default()).is_ok()); + + // Enabled without a view type is refused (measured 2026-10-24). + input.stream_specification = Some(StreamSpecification { + stream_enabled: true, + stream_view_type: None, + }); + let err = validate_create_table(&input, &LimitsConfig::default()).unwrap_err(); + assert_eq!( + err.to_string(), + "One or more parameter values were invalid: Table is being created with a stream enabled, UpdateViewType is required" + ); } #[test] diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index 67114a013..dadeb5fbf 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -489,7 +489,12 @@ fn apply_source_to_description( }) .collect() }), - stream_description: d.stream_specification.clone(), + // A disabled specification is kept by the backends for the + // closed stream's view type; the service's DescribeBackup for + // a table whose stream was disabled is not measured, so the + // member is omitted as it was before the specification was + // kept. + stream_description: d.stream_specification.clone().filter(|s| s.stream_enabled), time_to_live_description: d.time_to_live.clone(), sse_description: SseDescription::from_specification( d.sse_specification.as_ref(), diff --git a/crates/engine/src/create_table.rs b/crates/engine/src/create_table.rs index e9948c111..b490c2b2c 100755 --- a/crates/engine/src/create_table.rs +++ b/crates/engine/src/create_table.rs @@ -29,6 +29,7 @@ pub async fn handle_create_table( ], )?; + crate::require_stream_enabled_member(&body)?; let mut input: CreateTableInput = serde_json::from_value(body).map_err(|e| { let msg = e.to_string(); if msg.contains("validation error detected") diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 74083fbfa..18e87f1ec 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -156,6 +156,24 @@ pub(crate) fn sanitize_storage_error(e: extenddb_storage::error::StorageError) - /// Enum validation errors (produced by custom Deserialize impls) contain /// "validation error detected" and should be returned as `ValidationException`. /// All other deserialization failures are `SerializationException`. +/// A `StreamSpecification` member without `StreamEnabled`, on `CreateTable` +/// and `UpdateTable` alike (measured 2026-10-24). Checked before the body is +/// deserialised, where the missing field would otherwise surface as a +/// serialization error; its order against the other request checks is not +/// measured. +pub(crate) fn require_stream_enabled_member(body: &serde_json::Value) -> Result<(), DynamoDbError> { + if let Some(spec) = body.get("StreamSpecification").filter(|v| !v.is_null()) + && spec + .get("StreamEnabled") + .is_none_or(serde_json::Value::is_null) + { + return Err(DynamoDbError::ValidationException( + "1 validation error detected: Value null at 'streamSpecification.streamEnabled' failed to satisfy constraint: Member must not be null".to_owned(), + )); + } + Ok(()) +} + pub(crate) fn deserialize_error(e: serde_json::Error) -> DynamoDbError { let msg = e.to_string(); if msg.contains("validation error detected") diff --git a/crates/engine/src/streams.rs b/crates/engine/src/streams.rs index c8ba419bc..34305a850 100755 --- a/crates/engine/src/streams.rs +++ b/crates/engine/src/streams.rs @@ -27,6 +27,14 @@ pub async fn handle_describe_stream( ) -> Result { let input: DescribeStreamInput = serde_json::from_value(body) .map_err(|e| DynamoDbError::SerializationException(e.to_string()))?; + if let Some(start) = &input.exclusive_start_shard_id + && start.chars().count() < SHARD_ID_MIN_LEN + { + return Err(DynamoDbError::ValidationException(constraint( + "exclusiveStartShardId", + "have length greater than or equal to 28", + ))); + } let desc = ctx .storage @@ -51,6 +59,23 @@ pub async fn handle_list_streams( ) -> Result { let input: ListStreamsInput = serde_json::from_value(body) .map_err(|e| DynamoDbError::SerializationException(e.to_string()))?; + // Measured 2026-10-24: Limit=0 and a start ARN under 37 characters are + // refused with the model's constraint sentences (no value quoted). The + // upper bound of Limit is not measured; it is clamped to 100. + if input.limit.is_some_and(|l| l < 1) { + return Err(DynamoDbError::ValidationException(constraint( + "limit", + "have value greater than or equal to 1", + ))); + } + if let Some(start) = &input.exclusive_start_stream_arn + && start.chars().count() < 37 + { + return Err(DynamoDbError::ValidationException(constraint( + "exclusiveStartStreamArn", + "have length greater than or equal to 37", + ))); + } let limit = input.limit.unwrap_or(100).min(100); let (streams, last_arn) = ctx @@ -86,6 +111,33 @@ pub async fn handle_get_shard_iterator( let input: GetShardIteratorInput = serde_json::from_value(body) .map_err(|e| DynamoDbError::SerializationException(e.to_string()))?; + // The service's ShardId bound is 28 to 65 characters, checked before any + // lookup with the model's constraint sentences (both measured 2026-10-24). + // A SequenceNumber, when given, is at least 21 characters (measured with + // "0" on AT_SEQUENCE_NUMBER). + let shard_len = input.shard_id.chars().count(); + if shard_len < SHARD_ID_MIN_LEN { + return Err(DynamoDbError::ValidationException( + SHARD_ID_TOO_SHORT.to_owned(), + )); + } + if shard_len > SHARD_ID_MAX_LEN { + return Err(DynamoDbError::ValidationException(constraint( + "shardId", + "have length less than or equal to 65", + ))); + } + if input + .sequence_number + .as_ref() + .is_some_and(|n| n.chars().count() < 21) + { + return Err(DynamoDbError::ValidationException(constraint( + "sequenceNumber", + "have length greater than or equal to 21", + ))); + } + // Validate the stream and shard exist before issuing an iterator. ctx.storage .validate_shard(&ctx.account_id, &input.stream_arn, &input.shard_id) @@ -106,9 +158,12 @@ pub async fn handle_get_shard_iterator( ShardIteratorType::AtSequenceNumber => { // Convert to AFTER_SEQUENCE_NUMBER by subtracting 1, so the // exclusive "after" semantics produce inclusive "at" behavior. + // Measured 2026-10-24 for AT_SEQUENCE_NUMBER; AFTER_SEQUENCE_NUMBER + // without a number is not measured and keeps its sentence below. let raw = input.sequence_number.clone().ok_or_else(|| { DynamoDbError::ValidationException( - "SequenceNumber is required for AT_SEQUENCE_NUMBER iterator type".to_owned(), + "One or more parameter values were invalid: SequenceNumber cannot be null" + .to_owned(), ) })?; let n = raw.parse::().map_err(|_| { @@ -164,6 +219,10 @@ pub async fn handle_get_shard_iterator( /// Shard iterator expiration: 15 minutes (900 seconds), matching real `DynamoDB`. const SHARD_ITERATOR_EXPIRY_SECS: u64 = 900; +/// The shortest and longest `ShardId` the service accepts. +const SHARD_ID_MIN_LEN: usize = 28; +const SHARD_ID_MAX_LEN: usize = 65; + /// The model's constraint sentence for a member value, as the service renders /// it for the streams API (no value quoted; measured 2026-10-24 on `shardId`, /// `exclusiveStartShardId`, `sequenceNumber`, `limit` and @@ -174,6 +233,10 @@ fn constraint(member: &str, rule: &str) -> String { ) } +/// `GetShardIterator` with a `ShardId` under 28 characters (measured 2026-10-24 +/// with `nope`); the same shape `constraint` renders. +pub const SHARD_ID_TOO_SHORT: &str = "1 validation error detected: Value at 'shardId' failed to satisfy constraint: Member must have length greater than or equal to 28"; + /// The message `DynamoDB` returns for an expired shard iterator (measured live, /// `live-r2/iter-live.json`: an iterator 990 seconds old returns /// `ExpiredIteratorException` with exactly this text). Shared so the @@ -523,6 +586,18 @@ mod tests { base64::Engine::encode(&base64::engine::general_purpose::STANDARD, s) } + #[test] + fn the_constraint_sentences_match_the_measured_ones() { + assert_eq!( + super::constraint("shardId", "have length greater than or equal to 28"), + super::SHARD_ID_TOO_SHORT + ); + assert_eq!( + super::constraint("limit", "have value greater than or equal to 1"), + "1 validation error detected: Value at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ); + } + #[test] fn classify_current_token_is_stream() { let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; diff --git a/crates/engine/src/update_table.rs b/crates/engine/src/update_table.rs index 6db763f82..ed49a6990 100755 --- a/crates/engine/src/update_table.rs +++ b/crates/engine/src/update_table.rs @@ -45,6 +45,7 @@ pub async fn handle_update_table( ], )?; + crate::require_stream_enabled_member(&body)?; let mut input: UpdateTableInput = serde_json::from_value(body).map_err(crate::deserialize_error)?; input.resolve_table_throughput_mode(); @@ -112,6 +113,16 @@ pub async fn handle_update_table( "StreamViewType must be specified when StreamEnabled is true".to_owned(), )); } + // A disable carries no view type (measured 2026-10-24 on a never-streamed + // table and on an enabled one alike; checked before the no-stream rule). + if let Some(spec) = &input.stream_specification + && !spec.stream_enabled + && spec.stream_view_type.is_some() + { + return Err(DynamoDbError::ValidationException( + "One or more parameter values were invalid: If stream is being disabled, then UpdateViewType must not be specified".to_owned(), + )); + } // Switching to PROVISIONED requires explicit throughput values. if matches!(input.billing_mode, Some(BillingMode::Provisioned)) @@ -142,6 +153,40 @@ pub async fn handle_update_table( validate_gsi_updates(&input)?; + // A stream change travels alone. Measured 2026-10-24: beside BillingMode or + // ProvisionedThroughput the service refuses with the IOPS sentence, beside + // any GlobalSecondaryIndexUpdates entry with the index sentence, and + // applies nothing. The per-index definition checks above come first, and + // the already-enabled refusal (`Table already has an enabled stream`) comes + // before the IOPS sentence, so the table's stored specification is read + // here when a companion change is present. DeletionProtectionEnabled beside + // a stream change is refused with the service's only-operation sentence + // (measured 2026-10-24); the general rule for DeletionProtection beside any + // other change is PR #474's and is not claimed here. TableClass and + // OnDemandThroughput beside a stream change are not measured and stay + // accepted. + if let Some(companion) = stream_change_companion(&input) { + let existing = ctx + .storage + .describe_table( + &ctx.account_id, + extenddb_core::types::DescribeTableInput { + table_name: input.table_name.clone(), + }, + ) + .await + .map_err(|e| update_table_err_to_dynamo(e, &input.table_name))?; + if let Some(spec) = &input.stream_specification { + extenddb_storage::stream_lineage::stream_change( + &input.table_name, + existing.stream_specification.as_ref(), + spec, + ) + .map_err(|e| update_table_err_to_dynamo(e, &input.table_name))?; + } + return Err(DynamoDbError::ValidationException(companion.to_owned())); + } + // A created GSI's projected attributes count toward the table's limit // together with the indexes it already has: the total CreateTable checks, // and so the total a restore of this table's backups checks. @@ -203,6 +248,37 @@ pub async fn handle_update_table( serialize_output(&output) } +/// The sentence the service refuses a stream change with when the same +/// `UpdateTable` also changes the table's billing or throughput, creates or +/// deletes an index, or changes deletion protection. `None` when the request +/// carries no stream change or no such companion. With both the billing and +/// the index companion present the IOPS sentence is answered (measured); the +/// deletion-protection sentence's order against the other two is not measured +/// and comes last. +fn stream_change_companion(input: &UpdateTableInput) -> Option<&'static str> { + input.stream_specification.as_ref()?; + if input.billing_mode.is_some() || input.provisioned_throughput.is_some() { + return Some( + "One or more parameter values were invalid: You cannot modify stream status while updating table IOPS", + ); + } + if input + .global_secondary_index_updates + .as_ref() + .is_some_and(|u| !u.is_empty()) + { + return Some( + "One or more parameter values were invalid: You cannot create or delete index while changing stream status", + ); + } + if input.deletion_protection_enabled.is_some() { + return Some( + "One or more parameter values were invalid: DeletionProtection modification must be the only operation in the request", + ); + } + None +} + /// The per-entry rules for `GlobalSecondaryIndexUpdates`, checked before any /// storage read: each entry has exactly one of Create, Update, or Delete, /// and a Create has a valid name, a key schema of defined attributes, and @@ -347,157 +423,5 @@ fn update_table_err_to_dynamo( } #[cfg(test)] -mod tests { - use super::*; - use extenddb_storage::error::StorageError; - - #[test] - fn a_created_gsi_throughput_below_one_is_refused() { - let input = |pt: Value| -> UpdateTableInput { - serde_json::from_value(serde_json::json!({ - "TableName": "t", - "AttributeDefinitions": [{ "AttributeName": "g", "AttributeType": "S" }], - "GlobalSecondaryIndexUpdates": [{ "Create": { - "IndexName": "gix", - "KeySchema": [{ "AttributeName": "g", "KeyType": "HASH" }], - "Projection": { "ProjectionType": "ALL" }, - "ProvisionedThroughput": pt, - }}], - })) - .unwrap() - }; - for (read, write) in [(0, 0), (0, 1), (1, 0), (-1, 5)] { - let pt = serde_json::json!({ "ReadCapacityUnits": read, "WriteCapacityUnits": write }); - match validate_gsi_updates(&input(pt)).unwrap_err() { - DynamoDbError::ValidationException(m) => { - assert!(m.ends_with("for index: gix"), "{m}"); - } - other => panic!("({read}, {write}): {other:?}"), - } - } - let ok = serde_json::json!({ "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }); - validate_gsi_updates(&input(ok)).unwrap(); - } - - #[test] - fn projected_total_counts_kept_indexes_and_created_ones() { - use extenddb_core::types::{ - GlobalSecondaryIndexUpdate, GsiDescription, LsiDescription, Projection, ProjectionType, - TableDescription, - }; - let include = |n: usize| Projection { - projection_type: ProjectionType::Include, - non_key_attributes: Some((0..n).map(|i| format!("a{i}")).collect()), - }; - let existing = TableDescription { - global_secondary_indexes: Some(vec![ - GsiDescription { - index_name: "kept".to_owned(), - key_schema: Vec::new(), - projection: include(30), - index_status: "ACTIVE".to_owned(), - provisioned_throughput: None, - index_size_bytes: 0, - item_count: 0, - index_arn: String::new(), - }, - GsiDescription { - index_name: "dropped".to_owned(), - key_schema: Vec::new(), - projection: include(50), - index_status: "ACTIVE".to_owned(), - provisioned_throughput: None, - index_size_bytes: 0, - item_count: 0, - index_arn: String::new(), - }, - ]), - local_secondary_indexes: Some(vec![LsiDescription { - index_name: "lsi".to_owned(), - key_schema: Vec::new(), - projection: include(20), - index_size_bytes: 0, - item_count: 0, - index_arn: String::new(), - }]), - ..Default::default() - }; - let create = |n: usize| -> GlobalSecondaryIndexUpdate { - serde_json::from_value(serde_json::json!({ "Create": { - "IndexName": "new", "KeySchema": [], - "Projection": serde_json::to_value(include(n)).unwrap(), - }})) - .unwrap() - }; - let delete: GlobalSecondaryIndexUpdate = - serde_json::from_value(serde_json::json!({ "Delete": { "IndexName": "dropped" } })) - .unwrap(); - - assert_eq!(projected_after_gsi_updates(&existing, &[create(1)]), 101); - assert_eq!( - projected_after_gsi_updates(&existing, &[create(1), delete]), - 51 - ); - assert!( - extenddb_core::validation::validate_projected_attribute_total( - projected_after_gsi_updates(&existing, &[create(1)]) - ) - .is_err() - ); - } - - /// A backend that cannot serve the request reports a 400, not a 500. - /// - /// This arm was missing while CreateTable had it, so the same refusal - /// answered differently depending on which operation carried it. It matters - /// for a backend whose vector capability is decided at runtime rather than at - /// compile time: its refusal arrives through this path. - #[test] - fn an_unsupported_feature_is_a_validation_exception() { - let err = update_table_err_to_dynamo( - StorageError::Unsupported("Vector indexes are not supported here".to_owned()), - "t", - ); - match err { - DynamoDbError::ValidationException(msg) => { - assert_eq!(msg, "Vector indexes are not supported here"); - } - other => panic!("expected ValidationException, got {other:?}"), - } - } - - /// The backend's whole message survives, unwrapped and unprefixed, because - /// the service's own wording for this case names both the table and the index - /// and no layer above the backend knows either. - #[test] - fn a_resource_in_use_refusal_keeps_the_backend_message() { - let measured = extenddb_core::types::vector_index_delete_in_allocation_phase("t", "vidx"); - let err = update_table_err_to_dynamo(StorageError::IndexesInUse(measured.clone()), "t"); - match err { - DynamoDbError::ResourceInUseException(msg) => assert_eq!(msg, measured), - other => panic!("expected ResourceInUseException, got {other:?}"), - } - } - - /// The measured whole string, byte for byte, from probe P2 on 2026-08-19. - #[test] - fn the_allocation_phase_refusal_is_the_measured_whole_string() { - assert_eq!( - extenddb_core::types::vector_index_delete_in_allocation_phase( - "eddbprobe-backfill", - "vidx2" - ), - "Attempt to change a resource which is still in use: Index creation is in resource \ - allocation phase. Retry deletion during backfilling phase or when the index is \ - active. Table: eddbprobe-backfill Index: vidx2" - ); - } - - /// A genuine fault still reports a 500 and is still logged as one. - #[test] - fn an_internal_failure_is_still_an_internal_server_error() { - let err = - update_table_err_to_dynamo(StorageError::Internal("disk on fire".to_owned()), "t"); - assert!(matches!(err, DynamoDbError::InternalServerError(_))); - } -} +#[path = "update_table_tests.rs"] +mod tests; diff --git a/crates/engine/src/update_table_tests.rs b/crates/engine/src/update_table_tests.rs new file mode 100644 index 000000000..642a95a26 --- /dev/null +++ b/crates/engine/src/update_table_tests.rs @@ -0,0 +1,215 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the `UpdateTable` request checks. + +use super::*; +use extenddb_storage::error::StorageError; + +/// Measured 2026-10-24: a stream change beside a billing or throughput +/// change, beside an index create or delete, or beside a deletion protection +/// change is refused with its own sentence; alone it is not. +#[test] +fn a_stream_change_travels_alone() { + let with = |extra: Value| -> UpdateTableInput { + let mut body = serde_json::json!({ + "TableName": "t", + "StreamSpecification": {"StreamEnabled": false}, + }); + for (k, v) in extra.as_object().unwrap() { + body[k] = v.clone(); + } + serde_json::from_value(body).unwrap() + }; + let iops = "One or more parameter values were invalid: You cannot modify stream status while updating table IOPS"; + let index = "One or more parameter values were invalid: You cannot create or delete index while changing stream status"; + assert_eq!( + stream_change_companion(&with(serde_json::json!({"BillingMode": "PROVISIONED"}))), + Some(iops) + ); + assert_eq!( + stream_change_companion(&with(serde_json::json!({ + "ProvisionedThroughput": {"ReadCapacityUnits": 5, "WriteCapacityUnits": 5} + }))), + Some(iops) + ); + assert_eq!( + stream_change_companion(&with(serde_json::json!({ + "GlobalSecondaryIndexUpdates": [{"Delete": {"IndexName": "g"}}] + }))), + Some(index) + ); + assert_eq!( + stream_change_companion(&with(serde_json::json!({ + "GlobalSecondaryIndexUpdates": [{"Create": { + "IndexName": "g", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"} + }}] + }))), + Some(index) + ); + assert_eq!(stream_change_companion(&with(serde_json::json!({}))), None); + assert_eq!( + stream_change_companion(&with( + serde_json::json!({"DeletionProtectionEnabled": true}) + )), + Some( + "One or more parameter values were invalid: DeletionProtection modification must be the only operation in the request" + ) + ); + let no_stream: UpdateTableInput = + serde_json::from_value(serde_json::json!({"TableName": "t", "BillingMode": "PROVISIONED"})) + .unwrap(); + assert_eq!(stream_change_companion(&no_stream), None); +} + +#[test] +fn a_created_gsi_throughput_below_one_is_refused() { + let input = |pt: Value| -> UpdateTableInput { + serde_json::from_value(serde_json::json!({ + "TableName": "t", + "AttributeDefinitions": [{ "AttributeName": "g", "AttributeType": "S" }], + "GlobalSecondaryIndexUpdates": [{ "Create": { + "IndexName": "gix", + "KeySchema": [{ "AttributeName": "g", "KeyType": "HASH" }], + "Projection": { "ProjectionType": "ALL" }, + "ProvisionedThroughput": pt, + }}], + })) + .unwrap() + }; + for (read, write) in [(0, 0), (0, 1), (1, 0), (-1, 5)] { + let pt = serde_json::json!({ "ReadCapacityUnits": read, "WriteCapacityUnits": write }); + match validate_gsi_updates(&input(pt)).unwrap_err() { + DynamoDbError::ValidationException(m) => { + assert!(m.ends_with("for index: gix"), "{m}"); + } + other => panic!("({read}, {write}): {other:?}"), + } + } + let ok = serde_json::json!({ "ReadCapacityUnits": 1, "WriteCapacityUnits": 1 }); + validate_gsi_updates(&input(ok)).unwrap(); +} + +#[test] +fn projected_total_counts_kept_indexes_and_created_ones() { + use extenddb_core::types::{ + GlobalSecondaryIndexUpdate, GsiDescription, LsiDescription, Projection, ProjectionType, + TableDescription, + }; + let include = |n: usize| Projection { + projection_type: ProjectionType::Include, + non_key_attributes: Some((0..n).map(|i| format!("a{i}")).collect()), + }; + let existing = TableDescription { + global_secondary_indexes: Some(vec![ + GsiDescription { + index_name: "kept".to_owned(), + key_schema: Vec::new(), + projection: include(30), + index_status: "ACTIVE".to_owned(), + provisioned_throughput: None, + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }, + GsiDescription { + index_name: "dropped".to_owned(), + key_schema: Vec::new(), + projection: include(50), + index_status: "ACTIVE".to_owned(), + provisioned_throughput: None, + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }, + ]), + local_secondary_indexes: Some(vec![LsiDescription { + index_name: "lsi".to_owned(), + key_schema: Vec::new(), + projection: include(20), + index_size_bytes: 0, + item_count: 0, + index_arn: String::new(), + }]), + ..Default::default() + }; + let create = |n: usize| -> GlobalSecondaryIndexUpdate { + serde_json::from_value(serde_json::json!({ "Create": { + "IndexName": "new", "KeySchema": [], + "Projection": serde_json::to_value(include(n)).unwrap(), + }})) + .unwrap() + }; + let delete: GlobalSecondaryIndexUpdate = + serde_json::from_value(serde_json::json!({ "Delete": { "IndexName": "dropped" } })) + .unwrap(); + + assert_eq!(projected_after_gsi_updates(&existing, &[create(1)]), 101); + assert_eq!( + projected_after_gsi_updates(&existing, &[create(1), delete]), + 51 + ); + assert!( + extenddb_core::validation::validate_projected_attribute_total(projected_after_gsi_updates( + &existing, + &[create(1)] + )) + .is_err() + ); +} + +/// A backend that cannot serve the request reports a 400, not a 500. +/// +/// This arm was missing while CreateTable had it, so the same refusal +/// answered differently depending on which operation carried it. It matters +/// for a backend whose vector capability is decided at runtime rather than at +/// compile time: its refusal arrives through this path. +#[test] +fn an_unsupported_feature_is_a_validation_exception() { + let err = update_table_err_to_dynamo( + StorageError::Unsupported("Vector indexes are not supported here".to_owned()), + "t", + ); + match err { + DynamoDbError::ValidationException(msg) => { + assert_eq!(msg, "Vector indexes are not supported here"); + } + other => panic!("expected ValidationException, got {other:?}"), + } +} + +/// The backend's whole message survives, unwrapped and unprefixed, because +/// the service's own wording for this case names both the table and the index +/// and no layer above the backend knows either. +#[test] +fn a_resource_in_use_refusal_keeps_the_backend_message() { + let measured = extenddb_core::types::vector_index_delete_in_allocation_phase("t", "vidx"); + let err = update_table_err_to_dynamo(StorageError::IndexesInUse(measured.clone()), "t"); + match err { + DynamoDbError::ResourceInUseException(msg) => assert_eq!(msg, measured), + other => panic!("expected ResourceInUseException, got {other:?}"), + } +} + +/// The measured whole string, byte for byte, from probe P2 on 2026-08-19. +#[test] +fn the_allocation_phase_refusal_is_the_measured_whole_string() { + assert_eq!( + extenddb_core::types::vector_index_delete_in_allocation_phase( + "eddbprobe-backfill", + "vidx2" + ), + "Attempt to change a resource which is still in use: Index creation is in resource \ + allocation phase. Retry deletion during backfilling phase or when the index is \ + active. Table: eddbprobe-backfill Index: vidx2" + ); +} + +/// A genuine fault still reports a 500 and is still logged as one. +#[test] +fn an_internal_failure_is_still_an_internal_server_error() { + let err = update_table_err_to_dynamo(StorageError::Internal("disk on fire".to_owned()), "t"); + assert!(matches!(err, DynamoDbError::InternalServerError(_))); +} diff --git a/crates/storage-cassandra/src/create_table.rs b/crates/storage-cassandra/src/create_table.rs index 5628926b4..8cd0a55af 100644 --- a/crates/storage-cassandra/src/create_table.rs +++ b/crates/storage-cassandra/src/create_table.rs @@ -18,19 +18,19 @@ use crate::backup_engine::RestoreHeartbeat; impl CassandraEngine { /// Initialize stream shards for a table atomically. /// - /// Writes 4 shard rows into the account keyspace and updates - /// `catalog.tables.stream_label` in a single LOGGED BATCH. - /// Returns the stream label (ISO 8601 timestamp). + /// Sets `catalog.tables.stream_label` with an LWT, then writes the four + /// shard rows of the new stream into the account keyspace in one LOGGED + /// BATCH, each starting at zero. Returns the stream label. pub(crate) async fn init_stream_shards( &self, account_id: &str, table_name: &str, account_keyspace: &str, table_id: &str, + view_type: extenddb_core::types::StreamViewType, ) -> Result { - let label = extenddb_storage::util::new_stream_label(); + let label = extenddb_storage::stream_lineage::next_stream_label(None); let catalog_keyspace = self.catalog_keyspace(); - let now_ms = chrono::Utc::now().timestamp_millis(); // Update stream_label via LWT to avoid mixing plain/LWT writes on the // tables row (which was created with IF NOT EXISTS). See ADR-0021. @@ -52,24 +52,14 @@ impl CassandraEngine { return Err(StorageError::TableNotFound(table_name.to_owned())); } - // Insert 4 shard rows into account keyspace (plain inserts, separate batch) - let mut shard_statements = Vec::new(); - for i in 0..crate::stream_util::SHARDS_PER_STREAM { - let shard_id = format!("shardId-{table_id}-{i:012}"); - shard_statements.push(format!( - "INSERT INTO {account_keyspace}.stream_shards \ - (shard_id, table_id, starting_sequence_number, created_at) \ - VALUES ('{shard_id}', '{table_id}', '{}', {now_ms})", - crate::stream_util::ZERO_SEQUENCE - )); - } - - let batch = format!("BEGIN BATCH\n{}\nAPPLY BATCH", shard_statements.join(";\n")); - // Note: values are interpolated rather than bound because Cassandra LOGGED BATCH - // does not support parameterized statements spanning multiple tables. - // All interpolated values are server-generated (UUIDs, timestamps, label from chrono). - if let Err(e) = self.session.query(&batch).await { - tracing::error!("init_stream_shards batch: {e}"); + let starts = vec![ + crate::stream_util::ZERO_SEQUENCE.to_owned(); + extenddb_storage::stream_lineage::SHARDS_PER_STREAM as usize + ]; + if let Err(e) = self + .insert_stream_shards(account_keyspace, table_id, &label, view_type, &starts) + .await + { // Clear the label so ListStreams does not report a stream with no shards. let clear_cql = format!( "UPDATE {catalog_keyspace}.tables SET stream_label = null \ @@ -81,9 +71,7 @@ impl CassandraEngine { cdrs_tokio::query_values!(account_id, table_name, label.as_str()), ) .await; - return Err(StorageError::Internal(format!( - "Failed to initialize stream shards: {e}" - ))); + return Err(e); } Ok(label) @@ -128,6 +116,7 @@ impl CassandraEngine { ) -> Result<(TableDescription, Option), StorageError> { let schedule_activation = restore.is_none(); Self::validate_account_id(account_id)?; + let input = extenddb_storage::stream_lineage::without_disabled_specification(input); let table_id = restore.map_or_else(|| uuid::Uuid::new_v4().to_string(), |r| r.table_id.clone()); @@ -449,22 +438,23 @@ impl CassandraEngine { } // Initialize stream shards (if enabled) - let stream_label: Option = if input + let stream_label: Option = match input .stream_specification .as_ref() - .is_some_and(|s| s.stream_enabled) + .filter(|s| s.stream_enabled) + .and_then(|s| s.stream_view_type) { - Some( + Some(view_type) => Some( self.init_stream_shards( account_id, &input.table_name, &account_keyspace, &table_id, + view_type, ) .await?, - ) - } else { - None + ), + None => None, }; // Create data table in account keyspace diff --git a/crates/storage-cassandra/src/lib.rs b/crates/storage-cassandra/src/lib.rs index 7dbfe98c4..0cbd5fbe1 100644 --- a/crates/storage-cassandra/src/lib.rs +++ b/crates/storage-cassandra/src/lib.rs @@ -29,6 +29,7 @@ pub mod operations; pub(crate) mod propagation_hold; mod snapshot_engine; mod stream_engine; +mod stream_lifecycle; pub mod stream_util; pub mod table_engine; mod table_helpers; diff --git a/crates/storage-cassandra/src/stream_engine.rs b/crates/storage-cassandra/src/stream_engine.rs index 8c83b0927..d1d551aaa 100755 --- a/crates/storage-cassandra/src/stream_engine.rs +++ b/crates/storage-cassandra/src/stream_engine.rs @@ -4,16 +4,15 @@ //! `StreamEngine` trait implementation for `CassandraEngine`. use cdrs_tokio::types::IntoRustByName; -use extenddb_core::types::{ - SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, StreamSummary, - StreamViewType, -}; +use extenddb_core::types::{StreamDescription, StreamRecord, StreamSummary}; use extenddb_storage::StreamEngine; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage; use extenddb_storage::util::{parse_stream_arn, shard_in_stream, stream_arn}; use futures::future::BoxFuture; use crate::CassandraEngine; +use crate::stream_lifecycle::{physical_shard_id, shard_range_of}; impl CassandraEngine { fn account_keyspace_for(&self, account_id: &str) -> String { @@ -59,58 +58,6 @@ impl CassandraEngine { Ok(self.account_keyspace(&account_id)) } - - /// The table id and current stream label of table `table_name` within - /// `account_id` (a primary-key read of the catalog), or `None` when the - /// account has no table of that name. - async fn stream_table_row( - &self, - account_id: &str, - table_name: &str, - ) -> Result)>, StorageError> { - let catalog_keyspace = self.catalog_keyspace(); - let query = format!( - "SELECT table_id, stream_label FROM {catalog_keyspace}.tables \ - WHERE account_id = ? AND table_name = ?" - ); - let result = self - .session - .query_with_values(&query, cdrs_tokio::query_values!(account_id, table_name)) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - rows.into_iter() - .next() - .map(|row| { - let table_id: String = row - .get_r_by_name("table_id") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let label: Option = row.get_by_name("stream_label").ok().flatten(); - Ok((table_id, label)) - }) - .transpose() - } - - /// The table id of the stream `stream_arn` names within `account_id`, or - /// the "stream does not exist" error when no table of that account carries - /// that name and stream label. - async fn stream_table_id( - &self, - account_id: &str, - stream_arn: &str, - ) -> Result { - let (table_name, stream_label) = parse_stream_arn(stream_arn)?; - match self.stream_table_row(account_id, &table_name).await? { - Some((table_id, label)) if label.as_deref() == Some(stream_label.as_str()) => { - Ok(table_id) - } - _ => Err(extenddb_storage::util::stream_does_not_exist()), - } - } } impl StreamEngine for CassandraEngine { @@ -132,7 +79,7 @@ impl StreamEngine for CassandraEngine { }) } - #[allow(unused_variables)] // `after` and `start` are used in query_with_values below + #[allow(unused_variables)] // `after` is used in query_with_values below fn get_stream_records( &self, account_id: &str, @@ -153,33 +100,41 @@ impl StreamEngine for CassandraEngine { // account keyspace. The shard is the table's when its row carries // that table id, and the stream's when `shard_in_stream` says so // from the table's label and the shard id; otherwise (another - // table, another account, or an earlier stream label on the same - // table) the answer is the "stream does not exist" error and the - // records read are discarded. The keyspace follows from the caller's - // account, so no table id is parsed out of the shard id. + // table, another account, or a stream of the same table the shard + // is not part of) the answer is the "stream does not exist" error + // and the records read are discarded. The keyspace follows from the + // caller's account, so no table id is parsed out of the shard id. + // + // A shard's records are a range of its index's partition + // (`stream_lifecycle`): after its starting number and, once closed, + // up to its ending number. The partition follows from the shard id + // alone (`physical_shard_id`), so the records read starts at the + // iterator's position together with the shard row read, and the + // range the shard row gives is applied to the rows it returns. let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - let Some((table_id, table_label)) = - self.stream_table_row(&account_id, &table_name).await? - else { + let Some((table, _)) = self.stream_table_row(&account_id, &table_name).await? else { return Err(extenddb_storage::util::stream_does_not_exist()); }; let keyspace = self.account_keyspace_for(&account_id); + let physical = physical_shard_id(&table.table_id, &shard_id); - let shard_query = - format!("SELECT table_id FROM {keyspace}.stream_shards WHERE shard_id = ?"); + let shard_query = format!( + "SELECT table_id, starting_sequence_number, ending_sequence_number \ + FROM {keyspace}.stream_shards WHERE shard_id = ?" + ); let shard_read = self .session .query_with_values(&shard_query, cdrs_tokio::query_values!(shard_id.as_str())); - let query = if let Some(ref after) = after_sequence { + let query = if after_sequence.is_some() { format!( - "SELECT record_data FROM {keyspace}.stream_records \ + "SELECT sequence_number, record_data FROM {keyspace}.stream_records \ WHERE shard_id = ? AND sequence_number > ? \ LIMIT {limit}" ) } else { format!( - "SELECT record_data FROM {keyspace}.stream_records \ + "SELECT sequence_number, record_data FROM {keyspace}.stream_records \ WHERE shard_id = ? \ LIMIT {limit}" ) @@ -189,31 +144,42 @@ impl StreamEngine for CassandraEngine { self.session .query_with_values( &query, - cdrs_tokio::query_values!(shard_id.as_str(), after.as_str()), + cdrs_tokio::query_values!(physical.as_str(), after.as_str()), ) .await } else { self.session - .query_with_values(&query, cdrs_tokio::query_values!(shard_id.as_str())) + .query_with_values(&query, cdrs_tokio::query_values!(physical.as_str())) .await } }; let (shard_result, result) = futures::join!(shard_read, records_read); - let shard_table_id: Option = shard_result + let shard_row = shard_result .map_err(|e| StorageError::Internal(e.to_string()))? .response_body() .map_err(|e| StorageError::Internal(e.to_string()))? .into_rows() .unwrap_or_default() .into_iter() - .next() + .next(); + let shard_table_id: Option = shard_row + .as_ref() .and_then(|row| row.get_r_by_name("table_id").ok()); - if shard_table_id.as_deref() != Some(table_id.as_str()) - || !shard_in_stream(&table_id, &shard_id, table_label.as_deref(), &stream_label) - { - return Err(extenddb_storage::util::stream_does_not_exist()); - } + let range = match shard_row { + Some(row) + if shard_table_id.as_deref() == Some(table.table_id.as_str()) + && shard_in_stream( + &table.table_id, + &shard_id, + table.stream_label.as_deref(), + &stream_label, + ) => + { + shard_range_of(physical, &row) + } + _ => return Err(extenddb_storage::util::stream_does_not_exist()), + }; let rows = result .map_err(|e| StorageError::Internal(e.to_string()))? @@ -222,22 +188,42 @@ impl StreamEngine for CassandraEngine { .into_rows() .unwrap_or_default(); - let records: Vec = rows - .into_iter() - .map(|row| { - let data: String = row - .get_r_by_name("record_data") - .map_err(|e| StorageError::Internal(e.to_string()))?; - serde_json::from_str(&data).map_err(|e| StorageError::Internal(e.to_string())) - }) - .collect::>()?; + // The read started at the iterator's position, which is below the + // shard's start for a fresh iterator on a later stream: the rows up + // to the start are an earlier stream's and are skipped, and when no + // row of the shard follows them the next iterator starts at the + // shard's start. The rows past a closed shard's end are a later + // stream's. + let mut records: Vec = Vec::new(); + let mut skipped_to_start = false; + for row in rows { + let sequence: String = row + .get_r_by_name("sequence_number") + .map_err(|e| StorageError::Internal(e.to_string()))?; + if sequence <= range.after { + skipped_to_start = true; + continue; + } + if range.until.as_ref().is_some_and(|until| sequence > *until) { + break; + } + let data: String = row + .get_r_by_name("record_data") + .map_err(|e| StorageError::Internal(e.to_string()))?; + records.push( + serde_json::from_str(&data) + .map_err(|e| StorageError::Internal(e.to_string()))?, + ); + } - let last_seq = records.last().map(|r| r.dynamodb.sequence_number.clone()); + let last_seq = records + .last() + .map(|r| r.dynamodb.sequence_number.clone()) + .or_else(|| skipped_to_start.then(|| range.after.clone())); Ok((records, last_seq)) }) } - #[allow(unused_variables)] // `start` is used in query_with_values below fn describe_stream( &self, account_id: &str, @@ -249,155 +235,25 @@ impl StreamEngine for CassandraEngine { let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn_str)?; - let catalog_keyspace = self.catalog_keyspace(); - let account_keyspace = self.account_keyspace_for(&account_id); - - // Fetch table metadata from catalog - let query = format!( - "SELECT key_schema, stream_specification, table_status, table_id \ - FROM {catalog_keyspace}.tables \ - WHERE account_id = ? AND table_name = ? AND stream_label = ? \ - ALLOW FILTERING" - ); - let result = self - .session - .query_with_values( - &query, - cdrs_tokio::query_values!( - account_id.as_str(), - table_name.as_str(), - stream_label.as_str() - ), - ) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - - let row = rows - .into_iter() - .next() + let (table, ks_json) = self + .stream_table_row(&account_id, &table_name) + .await? .ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn_str))?; - - let ks_json: String = row - .get_r_by_name("key_schema") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let stream_spec_json: Option = - row.get_by_name("stream_specification").ok().flatten(); - let table_status: String = row - .get_r_by_name("table_status") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let table_id: String = row - .get_r_by_name("table_id") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let key_schema = serde_json::from_str(&ks_json) .map_err(|e| StorageError::Internal(e.to_string()))?; - - let stream_view_type = stream_spec_json - .and_then(|s| { - let v: serde_json::Value = serde_json::from_str(&s).ok()?; - serde_json::from_value(v["StreamViewType"].clone()).ok() - }) - .unwrap_or(StreamViewType::KeysOnly); - - // Fetch shards from account keyspace via secondary index on table_id - let limit_val = limit.unwrap_or(100); - let shard_query = if let Some(ref start) = exclusive_start_shard_id { - format!( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM {account_keyspace}.stream_shards \ - WHERE table_id = ? AND shard_id > ? \ - LIMIT {} ALLOW FILTERING", - limit_val + 1 - ) - } else { - format!( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM {account_keyspace}.stream_shards \ - WHERE table_id = ? \ - LIMIT {} ALLOW FILTERING", - limit_val + 1 - ) - }; - - let shard_result = if let Some(ref start) = exclusive_start_shard_id { - self.session - .query_with_values( - &shard_query, - cdrs_tokio::query_values!(table_id.as_str(), start.as_str()), - ) - .await - } else { - self.session - .query_with_values(&shard_query, cdrs_tokio::query_values!(table_id.as_str())) - .await - } - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let shard_rows = shard_result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit_val as usize; - let last_shard = if shard_rows.len() > limit_usize { - let row = &shard_rows[limit_usize - 1]; - let id: String = row - .get_r_by_name("shard_id") - .map_err(|e| StorageError::Internal(e.to_string()))?; - Some(id) - } else { - None - }; - - let shards: Vec = shard_rows - .into_iter() - .take(limit_usize) - .map(|row| { - let shard_id: String = row - .get_r_by_name("shard_id") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let parent_shard_id: Option = - row.get_by_name("parent_shard_id").ok().flatten(); - let starting: String = row - .get_r_by_name("starting_sequence_number") - .map_err(|e| StorageError::Internal(e.to_string()))?; - let ending: Option = - row.get_by_name("ending_sequence_number").ok().flatten(); - Ok(Shard { - shard_id, - parent_shard_id, - sequence_number_range: SequenceNumberRange { - starting_sequence_number: starting, - ending_sequence_number: ending, - }, - }) - }) - .collect::>()?; - - let stream_status = if table_status == "DELETING" { - StreamStatus::Disabling - } else { - StreamStatus::Enabled - }; - - Ok(StreamDescription { - stream_arn: stream_arn_str, - stream_label, - stream_status, - stream_view_type, - table_name, + let account_keyspace = self.account_keyspace_for(&account_id); + let shards = self + .stream_shard_rows(&account_keyspace, Some(&table.table_id)) + .await?; + stream_lineage::describe_stream( + stream_arn_str, + &table, key_schema, - shards, - last_evaluated_shard_id: last_shard, - }) + stream_label, + &shards, + limit, + exclusive_start_shard_id.as_deref(), + ) }) } @@ -412,75 +268,21 @@ impl StreamEngine for CassandraEngine { let table_name = table_name.map(str::to_string); let exclusive_start_stream_arn = exclusive_start_stream_arn.map(str::to_string); Box::pin(async move { - let catalog_keyspace = self.catalog_keyspace(); - - // Cassandra doesn't support IS NOT NULL filtering efficiently; fetch all and filter. - // For list_streams the result set is bounded by the number of tables per account. - let query = format!( - "SELECT table_name, stream_label FROM {catalog_keyspace}.tables \ - WHERE account_id = ? ALLOW FILTERING" - ); - let result = self - .session - .query_with_values(&query, cdrs_tokio::query_values!(account_id.as_str())) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - use cdrs_tokio::types::IntoRustByName; - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - - // Parse start cursor - let start_cursor = exclusive_start_stream_arn + let start = exclusive_start_stream_arn .as_deref() .map(parse_stream_arn) .transpose()?; - - let mut summaries: Vec = rows - .into_iter() - .filter_map(|row| { - let tn: String = row.get_r_by_name("table_name").ok()?; - let label: Option = row.get_by_name("stream_label").ok().flatten(); - let label = label?; // skip tables without streams - if let Some(ref filter_tn) = table_name - && &tn != filter_tn - { - return None; - } - Some(StreamSummary { - stream_arn: stream_arn(&self.region, &account_id, &tn, &label), - stream_label: label, - table_name: tn, - }) - }) - .collect(); - - // Sort for stable pagination (table_name, stream_label) - summaries.sort_by(|a, b| { - a.table_name - .cmp(&b.table_name) - .then(a.stream_label.cmp(&b.stream_label)) - }); - - // Apply cursor - if let Some((start_table, start_label)) = start_cursor { - summaries - .retain(|s| (&s.table_name, &s.stream_label) > (&start_table, &start_label)); - } - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - let last_arn = if summaries.len() > limit_usize { - summaries.get(limit_usize - 1).map(|s| s.stream_arn.clone()) - } else { - None - }; - summaries.truncate(limit_usize); - - Ok((summaries, last_arn)) + let tables = self.stream_table_rows(&account_id).await?; + let account_keyspace = self.account_keyspace_for(&account_id); + let shards = self.stream_shard_rows(&account_keyspace, None).await?; + stream_lineage::list_streams( + &tables, + &shards, + table_name.as_deref(), + limit, + start, + |t, l| stream_arn(&self.region, &account_id, t, l), + ) }) } @@ -537,32 +339,22 @@ impl StreamEngine for CassandraEngine { let stream_arn = stream_arn.to_string(); let shard_id = shard_id.to_string(); Box::pin(async move { - let table_id = self.stream_table_id(&account_id, &stream_arn).await?; - let account_keyspace = self.account_keyspace_for(&account_id); - - let shard_query = format!( - "SELECT shard_id FROM {account_keyspace}.stream_shards \ - WHERE shard_id = ? AND table_id = ? ALLOW FILTERING" - ); - let shard_result = self - .session - .query_with_values( - &shard_query, - cdrs_tokio::query_values!(shard_id.as_str(), table_id.as_str()), - ) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let found = shard_result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .is_some_and(|r| !r.is_empty()); - - if !found { - return Err(extenddb_storage::util::shard_not_in_stream(&stream_arn)); - } - Ok(()) + let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; + let table = self.stream_table_row(&account_id, &table_name).await?; + let shards = match &table { + Some((t, _)) => { + let account_keyspace = self.account_keyspace_for(&account_id); + self.stream_shard_rows(&account_keyspace, Some(&t.table_id)) + .await? + } + None => Vec::new(), + }; + stream_lineage::validate_shard( + table.as_ref().map(|(t, _)| t), + &stream_label, + &shards, + &shard_id, + ) }) } @@ -573,26 +365,8 @@ impl StreamEngine for CassandraEngine { let shard_id = shard_id.to_string(); Box::pin(async move { let keyspace = self.account_keyspace_for_shard(&shard_id).await?; - let query = format!( - "SELECT sequence_number FROM {keyspace}.stream_records \ - WHERE shard_id = ? ORDER BY sequence_number DESC LIMIT 1" - ); - let result = self - .session - .query_with_values(&query, cdrs_tokio::query_values!(shard_id.as_str())) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let rows = result - .response_body() - .map_err(|e| StorageError::Internal(e.to_string()))? - .into_rows() - .unwrap_or_default(); - - Ok(rows - .into_iter() - .next() - .map(|row| row.get_r_by_name("sequence_number").unwrap_or_default())) + let range = self.shard_range(&keyspace, &shard_id).await?; + self.last_sequence_in(&keyspace, &range).await }) } } diff --git a/crates/storage-cassandra/src/stream_lifecycle.rs b/crates/storage-cassandra/src/stream_lifecycle.rs new file mode 100644 index 000000000..aedd4816e --- /dev/null +++ b/crates/storage-cassandra/src/stream_lifecycle.rs @@ -0,0 +1,512 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stream lifecycle on Cassandra: closing a table's stream on +//! `UpdateTable StreamEnabled=false`, creating the next one, and reading the +//! table and shard rows the shared rules in `extenddb_storage::stream_lineage` +//! group into streams. +//! +//! The write paths compute a record's shard from the table id and the +//! partition key alone (`stream_util::assign_shard_id`), inside the LOGGED +//! BATCH of the data write, with no read of the shard rows. So records are +//! stored in one partition per shard index, `shardId-{table_id}-{index:012}`, +//! whatever stream is current, and a stream's shard is a range of that +//! partition: the records after its `StartingSequenceNumber`, up to its +//! `EndingSequenceNumber` once closed. A new stream's shard starts at a clock +//! value drawn when the stream is enabled, above every sequence issued before, +//! so a record belongs to at most one stream. The first stream of a table +//! starts at zero. + +use cdrs_tokio::types::IntoRustByName; +use extenddb_core::types::{StreamSpecification, StreamViewType}; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, enable_still_current, + next_stream_label, open_shard_ids_of, parse_shard_id, shard_id, stored_specification, + stream_change, +}; + +use crate::CassandraEngine; +use crate::cassandra_util::query_rows; +use crate::stream_util::ZERO_SEQUENCE; + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +/// The partition holding the records of `shard_id`: the table's partition for +/// the shard's index. A shard id this table did not issue maps to itself. +#[must_use] +pub(crate) fn physical_shard_id(table_id: &str, shard_id: &str) -> String { + parse_shard_id(table_id, shard_id).map_or_else( + || shard_id.to_owned(), + |(_, index)| format!("shardId-{table_id}-{index:012}"), + ) +} + +/// The table id a shard id names (`shardId-{table_id}-{digits}`). +pub(crate) fn shard_table_id(shard_id: &str) -> Option<&str> { + shard_id + .strip_prefix("shardId-")? + .rsplit_once('-') + .map(|(table_id, _)| table_id) +} + +/// The record range of one shard, for reading its partition. +pub(crate) struct ShardRange { + /// The partition its records are in. + pub physical: String, + /// Records after this sequence number belong to the shard. + pub after: String, + /// Records up to this one, once the shard is closed. + pub until: Option, +} + +/// The record range of the shard whose `stream_shards` row is `row` (with its +/// `starting_sequence_number` and `ending_sequence_number` columns), in the +/// partition `physical`. +pub(crate) fn shard_range_of(physical: String, row: &cdrs_tokio::types::rows::Row) -> ShardRange { + let start: Option = row.get_by_name("starting_sequence_number").ok().flatten(); + ShardRange { + physical, + after: start.unwrap_or_else(|| ZERO_SEQUENCE.to_owned()), + until: row.get_by_name("ending_sequence_number").ok().flatten(), + } +} + +/// The `tables` columns an `UpdateTable` stream change sets, as text. +pub(crate) type StreamColumns = Vec<(&'static str, String)>; + +/// The shard changes of an applied `UpdateTable` stream change: close the +/// open shards of the stream the change read (`close`, the table's label at +/// that time), then, on an enable, create the new stream's shards. +#[must_use] +pub(crate) struct PendingStream { + /// The specification JSON and label the change was decided on; the + /// catalog write is conditioned on both still being these values. + pub read_specification: Option, + pub read_label: Option, + pub close: Option, + pub open: Option<(String, StreamViewType)>, +} + +impl PendingStream { + /// Run the shard changes once the catalog row is written. + /// + /// The enable's shards are inserted after its catalog row is visible, so a + /// disable that reads the row in between passes its condition, writes the + /// disabled specification and closes the shards it finds, which may be + /// none or some of the four. The enable re-reads the table's stream fields + /// after its inserts and, when the row no longer says enabled with this + /// label, closes the shards it inserted. A disable written before the + /// re-read is seen by it; one written after finds all four rows open and + /// closes them itself; the two closes carry the same ending number. + pub(crate) async fn apply( + self, + engine: &CassandraEngine, + account_id: &str, + account_ks: &str, + table_name: &str, + table_id: &str, + ) -> Result<(), StorageError> { + if let Some(label) = &self.close { + engine + .close_stream_shards(account_ks, table_id, label) + .await?; + } + let Some((label, view_type)) = self.open else { + return Ok(()); + }; + // Each new shard starts at a sequence number drawn now from the same + // clock the writers draw from, so it is above every sequence issued so + // far. A write of the closed stream that commits after the close (its + // LOGGED BATCH drew its sequence before the ending number was taken) + // then falls between the old shard's ending number and the new + // shard's starting number: in neither stream, rather than in the new + // one under the old view type. + let start = engine + .hlc + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .generate(); + let starts: Vec = vec![start; SHARDS_PER_STREAM as usize]; + engine + .insert_stream_shards(account_ks, table_id, &label, view_type, &starts) + .await?; + let row = engine.stream_table_row(account_id, table_name).await?; + if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { + engine + .close_stream_shards(account_ks, table_id, &label) + .await?; + } + Ok(()) + } +} + +impl PendingStream { + /// Decide an `UpdateTable` `StreamSpecification` against the table row + /// fetched by `update_table_impl`: the catalog columns to set (as text) + /// and the shard changes to run once the row is written. + pub(crate) fn decide( + row: &cdrs_tokio::types::rows::Row, + table_name: &str, + requested: &StreamSpecification, + ) -> Result<(StreamColumns, Self), StorageError> { + let read_specification: Option = + row.get_by_name("stream_specification").ok().flatten(); + let current: Option = read_specification + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?; + let read_label: Option = row.get_by_name("stream_label").ok().flatten(); + let change = stream_change(table_name, current.as_ref(), requested)?; + let mut cols = vec![( + "stream_specification", + serde_json::to_string(&stored_specification(change, current.as_ref(), requested)) + .map_err(internal)?, + )]; + let open = match change { + StreamChange::Disable => None, + StreamChange::Enable(view_type) => { + let label = next_stream_label(read_label.as_deref()); + cols.push(("stream_label", label.clone())); + Some((label, view_type)) + } + }; + Ok(( + cols, + Self { + read_specification, + close: read_label.clone(), + read_label, + open, + }, + )) + } +} + +impl CassandraEngine { + /// The refusal for a stream change whose conditional catalog write found + /// the stream fields changed: re-read the row and decide the request + /// against it, which refuses the loser of two overlapping changes with the + /// ordinary sentence (`Table already has an enabled stream` or `Table has + /// no stream to disable`). When the re-read state would accept the request + /// (a disable overlapped an enable), the table is reported as not ACTIVE, + /// the nearest of ExtendDB's answers to the service's in-progress refusal. + pub(crate) async fn refuse_overlapping_stream_change( + &self, + account_id: &str, + table_name: &str, + input: &extenddb_core::types::UpdateTableInput, + ) -> Result<(), StorageError> { + let Some(spec) = &input.stream_specification else { + return Ok(()); + }; + let catalog_ks = self.catalog_keyspace(); + let row = query_rows::( + &self.session, + &format!( + "SELECT stream_specification FROM {catalog_ks}.tables \ + WHERE account_id = ? AND table_name = ?" + ), + cdrs_tokio::query_values!(account_id, table_name), + "update_table stream re-read", + ) + .await? + .into_iter() + .next() + .ok_or_else(|| StorageError::TableNotFound(table_name.to_owned()))?; + let current: Option = row.get_by_name("stream_specification").ok().flatten(); + let current: Option = current + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?; + stream_change(table_name, current.as_ref(), spec)?; + Err(StorageError::TableNotActive(table_name.to_owned())) + } + + /// Insert the four open shards of stream `label`, shard `i` starting at + /// `starts[i]`. The caller stores the label on the table row. + pub(crate) async fn insert_stream_shards( + &self, + account_ks: &str, + table_id: &str, + label: &str, + view_type: StreamViewType, + starts: &[String], + ) -> Result<(), StorageError> { + let now_ms = chrono::Utc::now().timestamp_millis(); + let mut statements = Vec::new(); + for (i, start) in (0..SHARDS_PER_STREAM).zip(starts) { + // Interpolated, not bound: a LOGGED BATCH here carries plain + // strings. Every value is server-generated (UUID table id, label + // digits, sequence digits). + statements.push(format!( + "INSERT INTO {account_ks}.stream_shards \ + (shard_id, table_id, starting_sequence_number, created_at) \ + VALUES ('{}', '{table_id}', '{start}', {now_ms})", + shard_id(table_id, label, view_type, i)? + )); + } + let batch = format!("BEGIN BATCH\n{}\nAPPLY BATCH", statements.join(";\n")); + self.session.query(&batch).await.map_err(|e| { + tracing::error!("insert_stream_shards batch: {e}"); + StorageError::Internal(format!("Failed to initialize stream shards: {e}")) + })?; + Ok(()) + } + + /// Close every open shard of the table: its `EndingSequenceNumber` becomes + /// the sequence number of its last record, or its starting number when it + /// holds none (both measured 2026-10-24). + pub(crate) async fn close_stream_shards( + &self, + account_ks: &str, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + let rows = self.stream_shard_rows(account_ks, Some(table_id)).await?; + let ids = open_shard_ids_of(table_id, label, &rows); + for row in rows { + if !ids.contains(&row.shard_id) { + continue; + } + let range = ShardRange { + physical: physical_shard_id(table_id, &row.shard_id), + after: row.starting_sequence_number.clone(), + until: None, + }; + let ending = self + .last_sequence_in(account_ks, &range) + .await? + .unwrap_or(row.starting_sequence_number); + crate::cassandra_util::execute::( + &self.session, + &format!( + "UPDATE {account_ks}.stream_shards SET ending_sequence_number = ? \ + WHERE shard_id = ?" + ), + cdrs_tokio::query_values!(ending.as_str(), row.shard_id.as_str()), + "close_stream_shards", + ) + .await?; + } + Ok(()) + } + + /// The sequence number of the last record in a shard's range. + pub(crate) async fn last_sequence_in( + &self, + account_ks: &str, + range: &ShardRange, + ) -> Result, StorageError> { + let rows = match &range.until { + Some(until) => { + query_rows::( + &self.session, + &format!( + "SELECT sequence_number FROM {account_ks}.stream_records \ + WHERE shard_id = ? AND sequence_number > ? AND sequence_number <= ? \ + ORDER BY sequence_number DESC LIMIT 1" + ), + cdrs_tokio::query_values!( + range.physical.as_str(), + range.after.as_str(), + until.as_str() + ), + "last_sequence_in", + ) + .await? + } + None => { + query_rows::( + &self.session, + &format!( + "SELECT sequence_number FROM {account_ks}.stream_records \ + WHERE shard_id = ? AND sequence_number > ? \ + ORDER BY sequence_number DESC LIMIT 1" + ), + cdrs_tokio::query_values!(range.physical.as_str(), range.after.as_str()), + "last_sequence_in", + ) + .await? + } + }; + rows.into_iter() + .next() + .map(|r| r.get_r_by_name("sequence_number").map_err(internal)) + .transpose() + } + + /// The record range of a shard, from its row. + pub(crate) async fn shard_range( + &self, + account_ks: &str, + shard_id: &str, + ) -> Result { + let table_id = shard_table_id(shard_id).unwrap_or_default(); + let physical = physical_shard_id(table_id, shard_id); + let row = query_rows::( + &self.session, + &format!( + "SELECT starting_sequence_number, ending_sequence_number \ + FROM {account_ks}.stream_shards WHERE shard_id = ?" + ), + cdrs_tokio::query_values!(shard_id), + "shard_range", + ) + .await? + .into_iter() + .next(); + Ok(match row { + Some(r) => shard_range_of(physical, &r), + None => ShardRange { + physical, + after: ZERO_SEQUENCE.to_owned(), + until: None, + }, + }) + } + + fn table_stream_row( + table_name: String, + row: &cdrs_tokio::types::rows::Row, + ) -> Result { + let spec: Option = row.get_by_name("stream_specification").ok().flatten(); + Ok(TableStreamRow { + table_name, + table_id: row.get_r_by_name("table_id").map_err(internal)?, + table_status: row.get_r_by_name("table_status").map_err(internal)?, + stream_label: row.get_by_name("stream_label").ok().flatten(), + stream_specification: spec + .as_deref() + .map(serde_json::from_str::) + .transpose() + .map_err(internal)?, + }) + } + + /// The table's stream columns and its key schema JSON, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let catalog_ks = self.catalog_keyspace(); + let row = query_rows::( + &self.session, + &format!( + "SELECT table_id, table_status, stream_label, stream_specification, key_schema \ + FROM {catalog_ks}.tables WHERE account_id = ? AND table_name = ?" + ), + cdrs_tokio::query_values!(account_id, table_name), + "stream_table_row", + ) + .await? + .into_iter() + .next(); + row.map(|r| { + Ok(( + Self::table_stream_row(table_name.to_owned(), &r)?, + r.get_r_by_name("key_schema").map_err(internal)?, + )) + }) + .transpose() + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + let catalog_ks = self.catalog_keyspace(); + let rows = query_rows::( + &self.session, + &format!( + "SELECT table_name, table_id, table_status, stream_label, stream_specification \ + FROM {catalog_ks}.tables WHERE account_id = ? ALLOW FILTERING" + ), + cdrs_tokio::query_values!(account_id), + "stream_table_rows", + ) + .await?; + let mut tables = Vec::new(); + for r in rows { + let label: Option = r.get_by_name("stream_label").ok().flatten(); + if label.is_none() { + continue; + } + let name: String = r.get_r_by_name("table_name").map_err(internal)?; + tables.push(Self::table_stream_row(name, &r)?); + } + Ok(tables) + } + + /// The shard rows of one table, or of every table in the account's + /// keyspace when `table_id` is `None`. + pub(crate) async fn stream_shard_rows( + &self, + account_ks: &str, + table_id: Option<&str>, + ) -> Result, StorageError> { + let columns = "table_id, shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number"; + let rows = match table_id { + Some(id) => { + query_rows::( + &self.session, + &format!("SELECT {columns} FROM {account_ks}.stream_shards WHERE table_id = ?"), + cdrs_tokio::query_values!(id), + "stream_shard_rows", + ) + .await? + } + None => { + query_rows::( + &self.session, + &format!("SELECT {columns} FROM {account_ks}.stream_shards"), + cdrs_tokio::query::QueryValues::SimpleValues(Vec::new()), + "stream_shard_rows", + ) + .await? + } + }; + rows.into_iter() + .map(|r| { + Ok(ShardRow { + table_id: r.get_r_by_name("table_id").map_err(internal)?, + shard_id: r.get_r_by_name("shard_id").map_err(internal)?, + parent_shard_id: r.get_by_name("parent_shard_id").ok().flatten(), + starting_sequence_number: r + .get_by_name("starting_sequence_number") + .ok() + .flatten() + .unwrap_or_else(|| ZERO_SEQUENCE.to_owned()), + ending_sequence_number: r.get_by_name("ending_sequence_number").ok().flatten(), + }) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const TID: &str = "550e8400-e29b-41d4-a716-446655440000"; + + #[test] + fn every_stream_of_a_table_reads_the_partition_of_its_shard_index() { + let label = "2026-10-24T08:54:32.805"; + for i in 0..SHARDS_PER_STREAM { + let labelled = shard_id(TID, label, StreamViewType::NewImage, i).unwrap(); + let legacy = format!("shardId-{TID}-{i:012}"); + assert_eq!(physical_shard_id(TID, &labelled), legacy); + assert_eq!(physical_shard_id(TID, &legacy), legacy); + assert_eq!(shard_table_id(&labelled), Some(TID)); + } + assert_eq!(physical_shard_id(TID, "shardId-other-1"), "shardId-other-1"); + } +} diff --git a/crates/storage-cassandra/src/update_table.rs b/crates/storage-cassandra/src/update_table.rs index 4a20b8618..ceb28b703 100644 --- a/crates/storage-cassandra/src/update_table.rs +++ b/crates/storage-cassandra/src/update_table.rs @@ -30,7 +30,7 @@ impl CassandraEngine { &self.session, &format!( "SELECT table_id, table_status, key_schema, attribute_definitions, \ - billing_mode, provisioned_throughput \ + billing_mode, provisioned_throughput, stream_specification, stream_label \ FROM {catalog_ks}.tables WHERE account_id = ? AND table_name = ?" ), cdrs_tokio::query_values!(account_id, input.table_name.as_str()), @@ -151,53 +151,17 @@ impl CassandraEngine { add_update!("on_demand_throughput", odt_json.as_str()); } - // Stream specification: add to batch, then handle shard init separately - // (init_stream_shards writes to a different keyspace and does its own batch). - let mut needs_shard_init = false; - let mut needs_label_restore = false; + // Stream specification: the catalog columns go in the tables LWT; the + // shard changes run after it (`PendingStream`), since the shards are + // in a different keyspace. + let mut pending_stream = None; if let Some(spec) = &input.stream_specification { - let spec_json = - serde_json::to_string(spec).map_err(|e| StorageError::Internal(e.to_string()))?; - add_update!("stream_specification", spec_json.as_str()); - - if spec.stream_enabled { - let account_ks = self.account_keyspace(account_id); - let existing = query_optional( - &self.session, - &format!( - "SELECT shard_id FROM {account_ks}.stream_shards \ - WHERE table_id = ? LIMIT 1 ALLOW FILTERING" - ), - cdrs_tokio::query_values!(table_id.as_str()), - "update_table stream_shards check", - ) - .await?; - - if existing.is_none() { - needs_shard_init = true; - } else { - // Re-enabling: check if stream_label needs restoring. - let label_row = query_optional( - &self.session, - &format!( - "SELECT stream_label FROM {catalog_ks}.tables \ - WHERE account_id = ? AND table_name = ?" - ), - cdrs_tokio::query_values!(account_id, input.table_name.as_str()), - "update_table stream_label check", - ) - .await?; - let has_label = label_row.is_some_and(|r| { - let v: Option = r.get_by_name("stream_label").ok().flatten(); - v.is_some() - }); - if !has_label { - needs_label_restore = true; - let label = extenddb_storage::util::new_stream_label(); - add_update!("stream_label", label.as_str()); - } - } + let (cols, pending) = + crate::stream_lifecycle::PendingStream::decide(&row, &input.table_name, spec)?; + for (col, val) in cols { + table_cols.push((col, Value::from(val.as_str()))); } + pending_stream = Some(pending); } // GSI updates: pre-validate, then add catalog changes to batch. @@ -416,8 +380,7 @@ impl CassandraEngine { table_cols, indexes_batch, indexes_batch_has_statements, - needs_shard_init, - needs_label_restore, + pending_stream, &table_id, &gsi_creates, &gsi_deletes, @@ -451,8 +414,7 @@ impl CassandraEngine { table_cols: Vec<(&'static str, Value)>, indexes_batch: BatchQueryBuilder, indexes_batch_has_statements: bool, - needs_shard_init: bool, - needs_label_restore: bool, + pending_stream: Option, table_id: &str, gsi_creates: &[(String, String)], gsi_deletes: &[String], @@ -505,13 +467,26 @@ impl CassandraEngine { .map(|(col, _)| format!("{col} = ?")) .collect::>() .join(", "); + // A stream change is decided on the specification and label the + // row carried and written only if they still are those values: + // of two overlapping stream changes the one whose write finds them + // changed re-reads the row and is refused by `stream_change` as a + // request arriving after the other's commit would be. + let condition = match &pending_stream { + Some(_) => "IF stream_specification = ? AND stream_label = ?", + None => "IF EXISTS", + }; let cql = format!( "UPDATE {catalog_ks}.tables SET {set_clause} \ - WHERE account_id = ? AND table_name = ? IF EXISTS" + WHERE account_id = ? AND table_name = ? {condition}" ); let mut vals: Vec = table_cols.into_iter().map(|(_, v)| v).collect(); vals.push(Value::from(account_id)); vals.push(Value::from(input.table_name.as_str())); + if let Some(pending) = &pending_stream { + vals.push(Value::from(pending.read_specification.as_deref())); + vals.push(Value::from(pending.read_label.as_deref())); + } let result = crate::cassandra_util::query_lwt( &self.session, &cql, @@ -536,7 +511,10 @@ impl CassandraEngine { } })?; // IF EXISTS returns false only if the row was deleted between our - // earlier SELECT and this UPDATE — treat as TableNotFound. + // earlier SELECT and this UPDATE: treat as TableNotFound. Under a + // stream change the condition also fails when another stream + // change committed in between; `refuse_overlapping_stream_change` + // re-reads the row and answers as that later request would be. // NOTE: the tables LWT commits before the indexes batch below runs. // A failure in the indexes batch leaves the table row updated with // no corresponding index rows; the recovery worker does not cover @@ -560,6 +538,10 @@ impl CassandraEngine { .await; }); } + if pending_stream.is_some() { + self.refuse_overlapping_stream_change(account_id, &input.table_name, input) + .await?; + } return Err(StorageError::TableNotFound(input.table_name.clone())); } } @@ -591,12 +573,12 @@ impl CassandraEngine { return Err(e); } - // Post-batch: shard init (has its own internal batch across two keyspaces). - if needs_shard_init { - self.init_stream_shards(account_id, &input.table_name, &account_ks, table_id) + // Post-batch: close the stream's shards and create the new stream's. + if let Some(pending) = pending_stream { + pending + .apply(self, account_id, &account_ks, &input.table_name, table_id) .await?; } - let _ = needs_label_restore; // stream_label is set via add_update! above // Post-batch: GSI data table DDL and async backfill. if let Some(updates) = &input.global_secondary_index_updates { diff --git a/crates/storage-cassandra/tests/direct/stream_disable.rs b/crates/storage-cassandra/tests/direct/stream_disable.rs new file mode 100644 index 000000000..6dd0a5406 --- /dev/null +++ b/crates/storage-cassandra/tests/direct/stream_disable.rs @@ -0,0 +1,389 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Disabling a table's stream closes its shards, and enabling it again +//! creates a new stream, against the rows measured on the service on +//! 2026-10-24. Records stay in one partition per shard index; each stream's +//! shard reads only its own range of it. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use extenddb_core::types::{ + AttributeDefinition, AttributeValue, CreateTableInput, DescribeStreamInput, DescribeTableInput, + KeySchemaElement, KeyType, ScalarAttributeType, StreamDescription, StreamSpecification, + StreamStatus, StreamViewType, TableDescription, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, TableEngine}; +use extenddb_storage_cassandra::CassandraEngine; + +use crate::helpers::setup_engine; + +async fn activate(engine: &CassandraEngine) { + tokio::time::sleep(Duration::from_millis(350)).await; + engine + .process_control_plane_transitions() + .await + .expect("process table transitions"); +} + +async fn describe_table(engine: &CassandraEngine, account: &str, name: &str) -> TableDescription { + engine + .describe_table( + account, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .unwrap() +} + +async fn describe_stream(engine: &CassandraEngine, account: &str, arn: &str) -> StreamDescription { + engine + .describe_stream( + account, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .unwrap() +} + +async fn set_stream( + engine: &CassandraEngine, + account: &str, + name: &str, + spec: serde_json::Value, +) -> Result { + let input = serde_json::json!({"TableName": name, "StreamSpecification": spec}); + let out = engine + .update_table(account, serde_json::from_value(input).unwrap()) + .await; + activate(engine).await; + out +} + +async fn put(engine: &CassandraEngine, account: &str, name: &str, pk: &str, view: StreamViewType) { + let key_info = engine.table_key_info(account, name).await.unwrap(); + let mut item = BTreeMap::new(); + item.insert("id".to_owned(), AttributeValue::S(pk.to_owned())); + let capture = StreamCapture { + view_type: view, + user_identity: None, + region: Arc::from("us-east-1"), + }; + engine + .put_item( + &key_info, + item, + false, + None, + &Default::default(), + Some(&capture), + ) + .await + .unwrap(); +} + +/// Every record a shard of the stream `arn` yields, from its start. +async fn read_all(engine: &CassandraEngine, account: &str, arn: &str, shard: &str) -> Vec { + let (records, _) = engine + .get_stream_records(account, arn, shard, None, 1000) + .await + .unwrap(); + records + .into_iter() + .map(|r| r.dynamodb.sequence_number) + .collect() +} + +#[tokio::test] +async fn disable_closes_every_shard_and_re_enable_creates_a_new_stream() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamDisable"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + for pk in ["k0", "k1", "k2"] { + put(&engine, &account, name, pk, StreamViewType::NewAndOldImages).await; + } + let enabled = describe_table(&engine, &account, name).await; + let old_arn = enabled.latest_stream_arn.clone().unwrap(); + + let disabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let mut rendered = disabled.clone(); + rendered.prepare_for_response(); + assert!(rendered.stream_specification.is_none()); + assert_eq!( + disabled.latest_stream_arn.as_deref(), + Some(old_arn.as_str()) + ); + let mut table = describe_table(&engine, &account, name).await; + table.prepare_for_response(); + assert!(table.stream_specification.is_none()); + + let closed = describe_stream(&engine, &account, &old_arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::NewAndOldImages); + let mut before = 0; + for shard in &closed.shards { + let range = &shard.sequence_number_range; + let records = read_all(&engine, &account, &old_arn, &shard.shard_id).await; + before += records.len(); + let expected = records + .last() + .cloned() + .unwrap_or_else(|| range.starting_sequence_number.clone()); + assert_eq!( + range.ending_sequence_number.as_deref(), + Some(expected.as_str()) + ); + } + assert_eq!(before, 3); + + match set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + { + Err(StorageError::Validation(m)) => { + assert_eq!( + m, + format!("Table has no stream to disable: TableName: {name}") + ); + } + other => panic!("{other:?}"), + } + + let reenabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new_arn = reenabled.latest_stream_arn.clone().unwrap(); + assert_ne!(new_arn, old_arn); + let new = describe_stream(&engine, &account, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.stream_view_type, StreamViewType::KeysOnly); + assert!( + new.shards + .iter() + .all(|n| !closed.shards.iter().any(|o| o.shard_id == n.shard_id)) + ); + match set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ) + .await + { + Err(StorageError::Validation(m)) => { + assert_eq!( + m, + format!("Table already has an enabled stream: TableName: {name}") + ); + } + other => panic!("{other:?}"), + } + + // A write after the re-enable lands in the shared partition, and only the + // new stream's shard yields it. + put(&engine, &account, name, "after", StreamViewType::KeysOnly).await; + let mut old_after = 0; + for shard in &closed.shards { + old_after += read_all(&engine, &account, &old_arn, &shard.shard_id) + .await + .len(); + } + assert_eq!(old_after, 3); + let mut on_new = 0; + for shard in &new.shards { + on_new += read_all(&engine, &account, &new_arn, &shard.shard_id) + .await + .len(); + } + assert_eq!(on_new, 1); + // LATEST on a closed shard resolves to its last record, not to the new + // stream's. + for shard in &closed.shards { + let latest = engine + .latest_sequence_number(&shard.shard_id) + .await + .unwrap(); + let end = shard.sequence_number_range.ending_sequence_number.clone(); + if latest.is_some() { + assert_eq!(latest, end); + } + } + + let (listed, _) = engine + .list_streams(&account, Some(name), 100, None) + .await + .unwrap(); + let arns: Vec<&str> = listed.iter().map(|s| s.stream_arn.as_str()).collect(); + assert_eq!(arns, [new_arn.as_str(), old_arn.as_str()]); + match engine + .validate_shard(&account, &old_arn, &new.shards[0].shard_id) + .await + { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Shard does not exist"); + } + other => panic!("{other:?}"), + } +} + +/// Of two stream changes fired at the same instant, one is applied and the +/// other refused (the conditional catalog write); the table ends with one +/// enabled stream of four open shards and every other shard closed. +#[tokio::test] +async fn overlapping_stream_changes_apply_exactly_one() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamOverlap"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + let input = |spec: serde_json::Value| -> extenddb_core::types::UpdateTableInput { + serde_json::from_value(serde_json::json!({"TableName": name, "StreamSpecification": spec})) + .unwrap() + }; + let disable = serde_json::json!({"StreamEnabled": false}); + let enable = serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}); + for round in 0..6 { + let (a, b) = match round % 3 { + 0 => (disable.clone(), disable.clone()), + 1 => (enable.clone(), disable.clone()), + _ => (disable.clone(), enable.clone()), + }; + let (ra, rb) = tokio::join!( + engine.update_table(&account, input(a)), + engine.update_table(&account, input(b)), + ); + activate(&engine).await; + let ok = [&ra, &rb].iter().filter(|r| r.is_ok()).count(); + for r in [&ra, &rb] { + if let Err(e) = r { + assert!( + matches!( + e, + StorageError::Validation(_) | StorageError::TableNotActive(_) + ), + "{e:?}" + ); + } + } + assert_eq!( + ok, 1, + "round {round}: exactly one change applies: {ra:?} {rb:?}" + ); + + // Audit: one enabled stream with four open shards at most, every + // other shard closed. + let table = describe_table(&engine, &account, name).await; + let (listed, _) = engine + .list_streams(&account, Some(name), 100, None) + .await + .unwrap(); + let mut enabled = 0; + for s in &listed { + let d = describe_stream(&engine, &account, &s.stream_arn).await; + let open = d + .shards + .iter() + .filter(|x| x.sequence_number_range.ending_sequence_number.is_none()) + .count(); + if matches!(d.stream_status, StreamStatus::Enabled) { + enabled += 1; + assert_eq!((d.shards.len(), open), (4, 4), "{}", s.stream_arn); + assert_eq!( + table.latest_stream_arn.as_deref(), + Some(s.stream_arn.as_str()) + ); + } else { + assert_eq!(open, 0, "a disabled stream has every shard closed"); + } + } + let spec_enabled = table + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled); + assert_eq!(enabled, usize::from(spec_enabled)); + // Back to enabled for the next round. + let _ = engine.update_table(&account, input(enable.clone())).await; + activate(&engine).await; + } +} diff --git a/crates/storage-cassandra/tests/direct/streams.rs b/crates/storage-cassandra/tests/direct/streams.rs index 16c86540e..7fcdc04a5 100644 --- a/crates/storage-cassandra/tests/direct/streams.rs +++ b/crates/storage-cassandra/tests/direct/streams.rs @@ -406,7 +406,15 @@ async fn test_validate_shard_ok() { &key_info.table_name, &stream_label, ); - let shard_id = shard_for("any-pk", &key_info.table_id); + // The stream's own shard id for the partition the record lands in. + let index = crc32fast::hash(b"any-pk") % 4; + let shard_id = extenddb_storage::stream_lineage::shard_id( + &key_info.table_id, + &stream_label, + StreamViewType::NewAndOldImages, + index, + ) + .unwrap(); engine .validate_shard(&key_info.account_id, &arn, &shard_id) diff --git a/crates/storage-cassandra/tests/direct_integration.rs b/crates/storage-cassandra/tests/direct_integration.rs index 1df0244eb..50193c8fd 100644 --- a/crates/storage-cassandra/tests/direct_integration.rs +++ b/crates/storage-cassandra/tests/direct_integration.rs @@ -51,6 +51,8 @@ mod scan; mod settings_store; #[path = "direct/snapshot_engine.rs"] mod snapshot_engine; +#[path = "direct/stream_disable.rs"] +mod stream_disable; #[path = "direct/stream_ownership.rs"] mod stream_ownership; #[path = "direct/streams.rs"] diff --git a/crates/storage-mongodb/src/data_engine.rs b/crates/storage-mongodb/src/data_engine.rs index dd8f4cc9f..82fc8623d 100644 --- a/crates/storage-mongodb/src/data_engine.rs +++ b/crates/storage-mongodb/src/data_engine.rs @@ -2026,14 +2026,17 @@ impl MongoEngine { // ordering safe under contention — see // stream_engine::next_sequence_number_in_session for the full // rationale. - let shard_id = self + let Some(shard_id) = self .assign_shard_in_session( &key_info.account_id, &key_info.table_name, &pk_str, &mut *session, ) - .await?; + .await? + else { + return Ok(()); + }; let seq = self .next_sequence_number_in_session(&shard_id, &mut *session) .await?; diff --git a/crates/storage-mongodb/src/lib.rs b/crates/storage-mongodb/src/lib.rs index b27cc0baf..923fc4f0d 100644 --- a/crates/storage-mongodb/src/lib.rs +++ b/crates/storage-mongodb/src/lib.rs @@ -23,6 +23,7 @@ mod operations; pub mod pushdown; mod snapshot_engine; mod stream_engine; +mod stream_lifecycle; mod table_engine; mod ttl_worker; mod worker_store; diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index 097b3bb2a..b779d1fe4 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -13,22 +13,18 @@ use futures::TryStreamExt; use futures::future::BoxFuture; use mongodb::bson::DateTime as BsonDateTime; -use mongodb::bson::{self, Document, doc}; +use mongodb::bson::{Document, doc}; use mongodb::options::FindOptions; -use extenddb_core::types::{ - DescribeStreamInput, SequenceNumberRange, Shard, StreamDescription, StreamEventName, - StreamRecord, StreamStatus, StreamSummary, StreamViewType, -}; +use extenddb_core::types::{DescribeStreamInput, StreamDescription, StreamEventName, StreamRecord}; use extenddb_storage::StreamEngine; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage; use extenddb_storage::util::{parse_stream_arn, shard_in_stream, stream_arn}; use extenddb_storage::{StreamListResult, StreamRecordsResult}; use crate::MongoEngine; -const SHARDS_PER_STREAM: u32 = 4; - /// Map a `StreamEventName` to its DynamoDB wire-format string. /// /// DynamoDB Streams records use uppercase event names (`INSERT`, `MODIFY`, `REMOVE`). @@ -41,69 +37,7 @@ pub(crate) fn event_name_ddb_str(name: StreamEventName) -> &'static str { } } -/// Build the mongo `stream_shards.shard_id` for a given table_id + shard index. -/// -/// **Security invariant (RFC-0003 §5.3, §8.2):** shard_id must incorporate -/// the table's globally-unique `table_id` (a UUID), not the caller-visible -/// `table_name`. Table names are only unique per-account, so two accounts -/// creating "orders" tables would generate colliding shard_ids under a -/// name-derived scheme, letting one account's `GetRecords(shard_id)` read -/// the other's stream records. `table_id` is a per-table-instance UUID that -/// resets on `DeleteTable + CreateTable` — the recreated table gets fresh -/// shard_ids, so leftover stream records from the deleted table never -/// resurface either. -/// -/// UUIDs are not guessable, so an attacker cannot synthesize a shard_id -/// belonging to another tenant without first observing it (which itself -/// requires an authenticated path scoped to that tenant's account). -pub(crate) fn build_shard_id(table_id: &str, shard_index: u32) -> String { - format!("shardId-{table_id}-{shard_index:012}") -} - impl MongoEngine { - /// Initialize stream shards for a table. Only creates shard documents; - /// the caller is responsible for setting `stream_label` on the table doc. - /// - /// Uses the table's UUID (`table_id`), not `table_name`, in the shard_id - /// — see `build_shard_id` for the security rationale. - pub(crate) async fn init_stream_shards(&self, table_id: &str) -> Result<(), StorageError> { - let shards_coll = self.data_db.collection::("stream_shards"); - for i in 0..SHARDS_PER_STREAM { - let shard_id = build_shard_id(table_id, i); - let start_seq = format!("{:021}", 0); - let insert_res = shards_coll - .insert_one(doc! { - "shard_id": &shard_id, - "table_id": table_id, - "starting_sequence_number": &start_seq, - "created_at": BsonDateTime::now(), - }) - .await; - if let Err(e) = insert_res { - // Treat a duplicate `shard_id` as idempotent no-op: two - // concurrent `UpdateTable(stream_enabled=true)` calls - // can both observe "no shards yet" under snapshot - // isolation and both reach this insert; the unique - // index on `stream_shards.shard_id` (`bootstrapper.rs`) - // means one of them lands E11000. RFC-0003 §10.3 - // requires repeated `UpdateTable` calls with the same - // specification to not corrupt state — swallowing - // this E11000 makes the redundant call a no-op rather - // than a wire-visible 500. - let is_dup = matches!( - *e.kind, - mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( - mongodb::error::WriteError { code: 11000, .. } - )) - ); - if !is_dup { - return Err(StorageError::Internal(e.to_string())); - } - } - } - Ok(()) - } - /// Draw the next sequence number for a shard *inside* the given /// transaction session. /// @@ -153,13 +87,18 @@ impl MongoEngine { /// *inside* the given transaction session. Pairs with /// `next_sequence_number_in_session` so the shard set the write is /// routed to is read at the same snapshot as the sequence draw. + /// + /// Only the open shards count: they belong to the table's enabled + /// stream. `None` when there are none, because the stream was disabled + /// after the caller read the table's key info; the write then records + /// nothing. pub(crate) async fn assign_shard_in_session( &self, account_id: &str, table_name: &str, partition_key: &str, session: &mut mongodb::ClientSession, - ) -> Result { + ) -> Result, StorageError> { let tables_coll = self.catalog_db.collection::("tables"); let table_doc = tables_coll .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) @@ -172,7 +111,9 @@ impl MongoEngine { let shards_coll = self.data_db.collection::("stream_shards"); let opts = FindOptions::builder().sort(doc! { "shard_id": 1 }).build(); let mut cursor = shards_coll - .find(doc! { "table_id": table_id }) + .find( + doc! { "table_id": table_id, "ending_sequence_number": mongodb::bson::Bson::Null }, + ) .with_options(opts) .session(&mut *session) .await @@ -183,21 +124,18 @@ impl MongoEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; - if shard_docs.is_empty() { - return Err(StorageError::Internal(format!( - "No stream shards for table {table_name}" - ))); - } - let shard_ids: Vec<&str> = shard_docs .iter() .filter_map(|d| d.get_str("shard_id").ok()) .collect(); + if shard_ids.is_empty() { + return Ok(None); + } let hash = crc32fast::hash(partition_key.as_bytes()); #[allow(clippy::cast_possible_truncation)] let idx = (hash as usize) % shard_ids.len(); - Ok(shard_ids[idx].to_owned()) + Ok(Some(shard_ids[idx].to_owned())) } /// Delete every stream_shards document for a given table_id, and every @@ -392,114 +330,26 @@ impl StreamEngine for MongoEngine { let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; - - let tables_coll = self.catalog_db.collection::("tables"); - let table_doc = tables_coll - .find_one(doc! { - "_id": { "account_id": &account_id, "table_name": &table_name }, - "stream_label": &stream_label, - }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? + let (table, table_doc) = self + .stream_table_row(&account_id, &table_name) + .await? .ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn_val))?; - let key_schema = table_doc .get("key_schema") - .and_then(|b| bson::from_bson(b.clone()).ok()) + .and_then(|b| mongodb::bson::from_bson(b.clone()).ok()) .ok_or_else(|| StorageError::Internal("Missing key_schema".to_owned()))?; - - let stream_view_type = table_doc - .get("stream_specification") - .and_then(|b| { - let json: serde_json::Value = bson::from_bson(b.clone()).ok()?; - json.get("StreamViewType") - .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) - }) - .unwrap_or(StreamViewType::KeysOnly); - - let table_status = table_doc.get_str("table_status").unwrap_or("ACTIVE"); - let table_id = table_doc.get_str("table_id").unwrap_or_default(); - - let limit = limit.unwrap_or(100); - let shards_coll = self.data_db.collection::("stream_shards"); - - let filter = if let Some(ref start) = exclusive_start_shard_id { - doc! { - "table_id": table_id, - "shard_id": { "$gt": start }, - } - } else { - doc! { "table_id": table_id } - }; - - let opts = FindOptions::builder() - .sort(doc! { "shard_id": 1 }) - .limit(limit + 1) - .build(); - - let cursor = shards_coll - .find(filter) - .with_options(opts) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let shard_docs: Vec = cursor - .try_collect() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - let last_shard = if shard_docs.len() > limit_usize { - shard_docs.get(limit_usize - 1).and_then(|d| { - d.get_str("shard_id") - .ok() - .map(std::borrow::ToOwned::to_owned) - }) - } else { - None - }; - - let shards: Vec = shard_docs - .into_iter() - .take(limit_usize) - .filter_map(|d| { - Some(Shard { - shard_id: d.get_str("shard_id").ok()?.to_owned(), - parent_shard_id: d - .get_str("parent_shard_id") - .ok() - .map(std::borrow::ToOwned::to_owned), - sequence_number_range: SequenceNumberRange { - starting_sequence_number: d - .get_str("starting_sequence_number") - .ok()? - .to_owned(), - ending_sequence_number: d - .get_str("ending_sequence_number") - .ok() - .map(std::borrow::ToOwned::to_owned), - }, - }) - }) - .collect(); - - let stream_status = if table_status == "DELETING" { - StreamStatus::Disabling - } else { - StreamStatus::Enabled - }; - - Ok(StreamDescription { - stream_arn: stream_arn_val, - stream_label, - stream_status, - stream_view_type, - table_name, + let shards = self + .stream_shard_rows(std::slice::from_ref(&table.table_id)) + .await?; + stream_lineage::describe_stream( + stream_arn_val, + &table, key_schema, - shards, - last_evaluated_shard_id: last_shard, - }) + stream_label, + &shards, + limit, + exclusive_start_shard_id.as_deref(), + ) }) } @@ -515,73 +365,21 @@ impl StreamEngine for MongoEngine { let exclusive_start_stream_arn = exclusive_start_stream_arn.map(std::borrow::ToOwned::to_owned); Box::pin(async move { - let tables_coll = self.catalog_db.collection::("tables"); - - let mut filter = doc! { - "_id.account_id": &account_id, - "stream_label": { "$ne": null }, - }; - - if let Some(ref tn) = table_name { - filter.insert("_id.table_name", tn.as_str()); - } - - if let Some(ref start_arn) = exclusive_start_stream_arn { - let (start_table, start_label) = parse_stream_arn(start_arn)?; - if table_name.is_some() { - filter.insert("stream_label", doc! { "$gt": &start_label }); - } else { - filter.insert( - "$or", - bson::bson!([ - { "_id.table_name": { "$gt": &start_table } }, - { "_id.table_name": &start_table, "stream_label": { "$gt": &start_label } } - ]), - ); - } - } - - let opts = FindOptions::builder() - .sort(doc! { "_id.table_name": 1, "stream_label": 1 }) - .limit(limit + 1) - .build(); - - let cursor = tables_coll - .find(filter) - .with_options(opts) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let docs: Vec = cursor - .try_collect() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - - let summaries: Vec = docs - .iter() - .take(limit_usize) - .filter_map(|d| { - let id = d.get_document("_id").ok()?; - let tn = id.get_str("table_name").ok()?; - let label = d.get_str("stream_label").ok()?; - Some(StreamSummary { - stream_arn: stream_arn(&self.region, &account_id, tn, label), - stream_label: label.to_owned(), - table_name: tn.to_owned(), - }) - }) - .collect(); - - let last_arn = if docs.len() > limit_usize { - summaries.last().map(|s| s.stream_arn.clone()) - } else { - None - }; - - Ok((summaries, last_arn)) + let start = exclusive_start_stream_arn + .as_deref() + .map(parse_stream_arn) + .transpose()?; + let tables = self.stream_table_rows(&account_id).await?; + let ids: Vec = tables.iter().map(|t| t.table_id.clone()).collect(); + let shards = self.stream_shard_rows(&ids).await?; + stream_lineage::list_streams( + &tables, + &shards, + table_name.as_deref(), + limit, + start, + |t, l| stream_arn(&self.region, &account_id, t, l), + ) }) } @@ -622,8 +420,9 @@ impl StreamEngine for MongoEngine { let shards_coll = self.data_db.collection::("stream_shards"); let opts = FindOptions::builder().sort(doc! { "shard_id": 1 }).build(); + // Only the open shards: they belong to the table's enabled stream. let cursor = shards_coll - .find(doc! { "table_id": table_id }) + .find(doc! { "table_id": table_id, "ending_sequence_number": mongodb::bson::Bson::Null }) .with_options(opts) .await .map_err(|e| StorageError::Internal(e.to_string()))?; @@ -695,32 +494,20 @@ impl StreamEngine for MongoEngine { let shard_id = shard_id.to_owned(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn_val)?; - - let tables_coll = self.catalog_db.collection::("tables"); - let table_doc = tables_coll - .find_one(doc! { - "_id": { "account_id": &account_id, "table_name": &table_name }, - "stream_label": &stream_label, - }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let Some(table_doc) = table_doc else { - return Err(extenddb_storage::util::stream_does_not_exist()); + let table = self.stream_table_row(&account_id, &table_name).await?; + let shards = match &table { + Some((t, _)) => { + self.stream_shard_rows(std::slice::from_ref(&t.table_id)) + .await? + } + None => Vec::new(), }; - - let table_id = table_doc.get_str("table_id").unwrap_or_default(); - - let shards_coll = self.data_db.collection::("stream_shards"); - let exists = shards_coll - .find_one(doc! { "shard_id": &shard_id, "table_id": table_id }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if exists.is_none() { - return Err(extenddb_storage::util::shard_not_in_stream(&stream_arn_val)); - } - Ok(()) + stream_lineage::validate_shard( + table.as_ref().map(|(t, _)| t), + &stream_label, + &shards, + &shard_id, + ) }) } @@ -764,16 +551,28 @@ mod tests { assert_eq!(event_name_ddb_str(StreamEventName::Remove), "REMOVE"); } + const LABEL: &str = "2026-10-24T08:54:14.037"; + + fn shard(table_id: &str, index: u32) -> String { + stream_lineage::shard_id( + table_id, + LABEL, + extenddb_core::types::StreamViewType::KeysOnly, + index, + ) + .unwrap() + } + #[test] fn shard_id_embeds_table_id_not_table_name() { let table_id = "550e8400-e29b-41d4-a716-446655440000"; assert_eq!( - build_shard_id(table_id, 0), - "shardId-550e8400-e29b-41d4-a716-446655440000-000000000000" + shard(table_id, 0), + "shardId-550e8400-e29b-41d4-a716-446655440000-1792832054037100" ); assert_eq!( - build_shard_id(table_id, 3), - "shardId-550e8400-e29b-41d4-a716-446655440000-000000000003" + shard(table_id, 3), + "shardId-550e8400-e29b-41d4-a716-446655440000-1792832054037103" ); } @@ -785,20 +584,15 @@ mod tests { // shard. let table_a = "550e8400-e29b-41d4-a716-446655440000"; let table_b = "6ba7b810-9dad-11d1-80b4-00c04fd430c8"; - assert_ne!(build_shard_id(table_a, 0), build_shard_id(table_b, 0)); + assert_ne!(shard(table_a, 0), shard(table_b, 0)); } #[test] fn shard_id_format_is_stable_across_shards_of_same_table() { // Same table_id, different shard index → deterministic ordering. let table_id = "550e8400-e29b-41d4-a716-446655440000"; - let s0 = build_shard_id(table_id, 0); - let s1 = build_shard_id(table_id, 1); - let s2 = build_shard_id(table_id, 2); - let s3 = build_shard_id(table_id, 3); - assert!(s0 < s1); - assert!(s1 < s2); - assert!(s2 < s3); + let ids: Vec = (0..4).map(|i| shard(table_id, i)).collect(); + assert!(ids.windows(2).all(|w| w[0] < w[1])); } } diff --git a/crates/storage-mongodb/src/stream_lifecycle.rs b/crates/storage-mongodb/src/stream_lifecycle.rs new file mode 100644 index 000000000..42c01b436 --- /dev/null +++ b/crates/storage-mongodb/src/stream_lifecycle.rs @@ -0,0 +1,312 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stream lifecycle on MongoDB: creating a table's stream, closing it on +//! `UpdateTable StreamEnabled=false`, and reading the table and shard +//! documents the shared rules in `extenddb_storage::stream_lineage` group into +//! streams. +//! +//! The table document is in the catalog database and the shards and records +//! in the data database. `UpdateTable` sets the specification and label on the +//! table document first and changes the shards after that write +//! ([`PendingStream`]), the order the PostgreSQL backend uses for the same +//! two-database split. + +use futures::TryStreamExt; +use mongodb::bson::DateTime as BsonDateTime; +use mongodb::bson::{self, Bson, Document, doc}; +use mongodb::options::FindOptions; + +use extenddb_core::types::{StreamSpecification, StreamViewType}; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, enable_still_current, + next_stream_label, open_shard_ids_of, shard_id, stored_specification, stream_change, +}; + +use crate::MongoEngine; + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +fn spec_of(doc: &Document) -> Option { + doc.get("stream_specification") + .and_then(|b| bson::from_bson(b.clone()).ok()) +} + +fn table_row(table_name: &str, doc: &Document) -> TableStreamRow { + TableStreamRow { + table_name: table_name.to_owned(), + table_id: doc.get_str("table_id").unwrap_or_default().to_owned(), + table_status: doc.get_str("table_status").unwrap_or("ACTIVE").to_owned(), + stream_label: doc + .get_str("stream_label") + .ok() + .filter(|s| !s.is_empty()) + .map(str::to_owned), + stream_specification: spec_of(doc), + } +} + +/// The shard changes of an applied `UpdateTable` stream change: close the +/// open shards of the stream the change read (`close`, the table's label at +/// that time), then, on an enable, create the new stream's shards. +#[must_use] +pub(crate) struct PendingStream { + table_id: String, + close: Option, + open: Option<(String, StreamViewType)>, +} + +impl PendingStream { + /// Run the shard changes once the table document is written. + /// + /// The enable's shards are inserted after its table document is visible, + /// so a disable that reads the document in between passes its check, + /// writes the disabled specification and closes the shards it finds, which + /// may be none or some of the four. The enable re-reads the table's stream + /// fields after its inserts and, when the document no longer says enabled + /// with this label, closes the shards it inserted. A disable written + /// before the re-read is seen by it; one written after finds all four + /// shards open and closes them itself; both closing is idempotent through + /// the `ending_sequence_number: null` filter. + pub(crate) async fn apply( + self, + engine: &MongoEngine, + account_id: &str, + table_name: &str, + ) -> Result<(), StorageError> { + if let Some(label) = &self.close { + engine.close_stream_shards(&self.table_id, label).await?; + } + if let Some((label, view_type)) = self.open { + engine + .init_stream_shards(&self.table_id, &label, view_type) + .await?; + let row = engine.stream_table_row(account_id, table_name).await?; + if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { + engine.close_stream_shards(&self.table_id, &label).await?; + } + } + Ok(()) + } +} + +impl MongoEngine { + /// Insert the four open shards of stream `label`. The caller stores the + /// label on the table document. + pub(crate) async fn init_stream_shards( + &self, + table_id: &str, + label: &str, + view_type: StreamViewType, + ) -> Result<(), StorageError> { + let shards_coll = self.data_db.collection::("stream_shards"); + for i in 0..SHARDS_PER_STREAM { + let insert_res = shards_coll + .insert_one(doc! { + "shard_id": shard_id(table_id, label, view_type, i)?, + "table_id": table_id, + "starting_sequence_number": format!("{:021}", 0), + "created_at": BsonDateTime::now(), + }) + .await; + if let Err(e) = insert_res { + // A duplicate `shard_id` (E11000 on the unique index the + // bootstrapper creates) is a repeat of the same insert, from + // a retried request: keep the shard that is there. + let is_dup = matches!( + *e.kind, + mongodb::error::ErrorKind::Write(mongodb::error::WriteFailure::WriteError( + mongodb::error::WriteError { code: 11000, .. } + )) + ); + if !is_dup { + return Err(internal(e)); + } + } + } + Ok(()) + } + + /// Close every open shard of the table: its `EndingSequenceNumber` becomes + /// the sequence number of its last record, or its starting number when it + /// holds none (both measured 2026-10-24). + pub(crate) async fn close_stream_shards( + &self, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + let shards_coll = self.data_db.collection::("stream_shards"); + let records_coll = self.data_db.collection::("stream_records"); + let rows = self + .stream_shard_rows(std::slice::from_ref(&table_id.to_owned())) + .await?; + let ids = open_shard_ids_of(table_id, label, &rows); + if ids.is_empty() { + return Ok(()); + } + let open: Vec = shards_coll + .find(doc! { "shard_id": { "$in": &ids }, "ending_sequence_number": Bson::Null }) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + for shard in open { + let id = shard.get_str("shard_id").map_err(internal)?; + let last: Option = records_coll + .find(doc! { "shard_id": id }) + .with_options( + FindOptions::builder() + .sort(doc! { "sequence_number": -1 }) + .limit(1) + .build(), + ) + .await + .map_err(internal)? + .try_next() + .await + .map_err(internal)?; + let ending = match last + .as_ref() + .and_then(|d| d.get_str("sequence_number").ok()) + { + Some(seq) => seq.to_owned(), + None => shard + .get_str("starting_sequence_number") + .map_err(internal)? + .to_owned(), + }; + shards_coll + .update_one( + doc! { "shard_id": id, "ending_sequence_number": Bson::Null }, + doc! { "$set": { "ending_sequence_number": ending } }, + ) + .await + .map_err(internal)?; + } + Ok(()) + } + + /// Decide an `UpdateTable` `StreamSpecification` against the table + /// document and add its catalog fields to `update_doc`. + /// + /// Disabling sets the specification to null, so DescribeTable omits + /// `StreamSpecification`, and keeps `stream_label`, so `LatestStreamArn` + /// keeps naming the disabled stream. Enabling stores the specification and + /// a new label. The returned shard changes run once `update_doc` is + /// written. This backend has no transaction around UpdateTable, so two + /// concurrent stream changes can both pass the check. + pub(crate) fn prepare_stream_update( + update_doc: &mut Document, + table_doc: &Document, + table_name: &str, + requested: &StreamSpecification, + ) -> Result { + let table = table_row(table_name, table_doc); + let change = stream_change(table_name, table.stream_specification.as_ref(), requested)?; + update_doc.insert( + "stream_specification", + bson::to_bson(&stored_specification( + change, + table.stream_specification.as_ref(), + requested, + )) + .map_err(internal)?, + ); + let open = match change { + StreamChange::Disable => None, + StreamChange::Enable(view_type) => { + let label = next_stream_label(table.stream_label.as_deref()); + update_doc.insert("stream_label", &label); + Some((label, view_type)) + } + }; + Ok(PendingStream { + table_id: table.table_id, + close: table.stream_label, + open, + }) + } + + /// The table's stream fields and its document, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let doc = self + .catalog_db + .collection::("tables") + .find_one(doc! { "_id": { "account_id": account_id, "table_name": table_name } }) + .await + .map_err(internal)?; + Ok(doc.map(|d| (table_row(table_name, &d), d))) + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + let docs: Vec = self + .catalog_db + .collection::("tables") + .find(doc! { "_id.account_id": account_id, "stream_label": { "$ne": Bson::Null } }) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + Ok(docs + .iter() + .filter_map(|d| { + let name = d.get_document("_id").ok()?.get_str("table_name").ok()?; + Some(table_row(name, d)) + }) + .collect()) + } + + /// The shard documents of the given tables. + pub(crate) async fn stream_shard_rows( + &self, + table_ids: &[String], + ) -> Result, StorageError> { + if table_ids.is_empty() { + return Ok(Vec::new()); + } + let docs: Vec = self + .data_db + .collection::("stream_shards") + .find(doc! { "table_id": { "$in": table_ids } }) + .await + .map_err(internal)? + .try_collect() + .await + .map_err(internal)?; + Ok(docs + .iter() + .filter_map(|d| { + Some(ShardRow { + table_id: d.get_str("table_id").ok()?.to_owned(), + shard_id: d.get_str("shard_id").ok()?.to_owned(), + parent_shard_id: d.get_str("parent_shard_id").ok().map(str::to_owned), + starting_sequence_number: d + .get_str("starting_sequence_number") + .ok()? + .to_owned(), + ending_sequence_number: d + .get_str("ending_sequence_number") + .ok() + .map(str::to_owned), + }) + }) + .collect()) + } +} + +#[cfg(test)] +#[path = "stream_live_tests.rs"] +mod live_tests; diff --git a/crates/storage-mongodb/src/stream_live_tests.rs b/crates/storage-mongodb/src/stream_live_tests.rs new file mode 100644 index 000000000..30e44c153 --- /dev/null +++ b/crates/storage-mongodb/src/stream_live_tests.rs @@ -0,0 +1,270 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Live MongoDB tests for the stream lifecycle: a disable closes the stream, +//! an enable creates a new one, and of two overlapping stream changes exactly +//! one is applied. +//! +//! They need a replica set and read its URI from `EXTENDDB_TEST_MONGODB_URI` +//! (for example `mongodb://127.0.0.1:27482/?directConnection=true`). They are +//! `#[ignore]`d so `cargo test --lib` passes without one; run them with +//! `cargo test -p extenddb-storage-mongodb --lib -- --ignored live_stream`. +//! Each test uses its own catalog and data databases and drops them. + +use std::sync::Arc; + +use mongodb::bson::{Document, doc}; + +use extenddb_core::types::{ + AttributeDefinition, BillingMode, CreateTableInput, DescribeStreamInput, DescribeTableInput, + KeySchemaElement, KeyType, ScalarAttributeType, StreamSpecification, StreamStatus, + StreamViewType, TableDescription, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{StreamEngine, TableEngine}; + +use crate::MongoEngine; + +const ACCOUNT: &str = "123456789012"; + +fn uri() -> String { + std::env::var("EXTENDDB_TEST_MONGODB_URI").expect( + "set EXTENDDB_TEST_MONGODB_URI to a MongoDB replica set, e.g. \ + mongodb://127.0.0.1:27482/?directConnection=true", + ) +} + +async fn engine() -> MongoEngine { + let tag = uuid::Uuid::new_v4().simple().to_string(); + let client = mongodb::Client::with_uri_str(uri()).await.expect("client"); + let slots = crate::backup_engine::restore_server_slot_count(10); + let engine = MongoEngine { + client: client.clone(), + catalog_db: client.database(&format!("s{}_cat", &tag[..12])), + data_db: client.database(&format!("s{}_data", &tag[..12])), + region: "us-east-1".to_owned(), + gsi_cache: Arc::default(), + tx_read_concern: mongodb::options::ReadConcern::snapshot(), + backup_liveness: crate::backup_engine::BackupLiveness::default(), + restore_server_slot_count: slots, + restore_server_slots: Arc::new(tokio::sync::Semaphore::new(slots)), + restore_account_slots: crate::backup_engine::RestoreAccountSlots::default(), + }; + engine + .catalog_db + .collection::("settings") + .insert_one(doc! { "_id": "control_plane_delay_seconds", "value": "0" }) + .await + .expect("settings"); + engine +} + +async fn finish(engine: MongoEngine) { + engine.catalog_db.drop().await.expect("drop catalog"); + engine.data_db.drop().await.expect("drop data"); +} + +async fn stream_table(engine: &MongoEngine, name: &str) -> TableDescription { + engine + .create_table_impl( + ACCOUNT, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + billing_mode: Some(BillingMode::PayPerRequest), + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + false, + ) + .await + .expect("create") +} + +fn set_stream(name: &str, spec: serde_json::Value) -> extenddb_core::types::UpdateTableInput { + serde_json::from_value(serde_json::json!({"TableName": name, "StreamSpecification": spec})) + .expect("input") +} + +async fn describe_stream( + engine: &MongoEngine, + arn: &str, +) -> extenddb_core::types::StreamDescription { + engine + .describe_stream( + ACCOUNT, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .expect("describe stream") +} + +/// Of two stream changes fired at the same instant, one is applied and the +/// other refused; the table ends with one stream enabled and four open +/// shards, and every other shard closed. +async fn overlapping_pair( + engine: &MongoEngine, + name: &str, + a: serde_json::Value, + b: serde_json::Value, +) -> (usize, usize) { + let (ra, rb) = tokio::join!( + engine.update_table(ACCOUNT, set_stream(name, a)), + engine.update_table(ACCOUNT, set_stream(name, b)), + ); + let ok = [&ra, &rb].iter().filter(|r| r.is_ok()).count(); + let refused = [&ra, &rb] + .iter() + .filter(|r| { + matches!( + r, + Err(StorageError::Validation(_)) | Err(StorageError::TableNotActive(_)) + ) + }) + .count(); + for r in [&ra, &rb] { + if let Err(e) = r { + assert!( + matches!( + e, + StorageError::Validation(_) | StorageError::TableNotActive(_) + ), + "{e:?}" + ); + } + } + (ok, refused) +} + +async fn audit(engine: &MongoEngine, name: &str) { + let table = engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .expect("describe table"); + let (listed, _) = engine + .list_streams(ACCOUNT, Some(name), 100, None) + .await + .expect("list"); + let mut enabled = 0; + for s in &listed { + let d = describe_stream(engine, &s.stream_arn).await; + let open = d + .shards + .iter() + .filter(|x| x.sequence_number_range.ending_sequence_number.is_none()) + .count(); + match d.stream_status { + StreamStatus::Enabled => { + enabled += 1; + assert_eq!(d.shards.len(), 4, "{}", s.stream_arn); + assert_eq!(open, 4, "an enabled stream has every shard open"); + assert_eq!( + table.latest_stream_arn.as_deref(), + Some(s.stream_arn.as_str()) + ); + } + _ => assert_eq!(open, 0, "a disabled stream has every shard closed"), + } + } + let spec_enabled = table + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled); + assert_eq!(enabled, usize::from(spec_enabled)); +} + +#[tokio::test] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_stream_overlapping_changes_apply_exactly_one() { + let engine = engine().await; + let name = "overlap"; + stream_table(&engine, name).await; + let disable = serde_json::json!({"StreamEnabled": false}); + let enable = serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}); + let mut applied = 0; + let mut refused = 0; + for round in 0..10 { + // Alternate the pairs so both orders and both mixes are exercised. + let (a, b) = match round % 3 { + 0 => (disable.clone(), disable.clone()), + 1 => (enable.clone(), disable.clone()), + _ => (disable.clone(), enable.clone()), + }; + let (ok, no) = overlapping_pair(&engine, name, a, b).await; + assert_eq!( + (ok, no), + (1, 1), + "round {round}: exactly one change applies" + ); + applied += ok; + refused += no; + audit(&engine, name).await; + // Bring the table back to enabled for the next round. + let _ = engine + .update_table(ACCOUNT, set_stream(name, enable.clone())) + .await; + audit(&engine, name).await; + } + assert_eq!((applied, refused), (10, 10)); + finish(engine).await; +} + +#[tokio::test] +#[ignore = "needs a MongoDB replica set in EXTENDDB_TEST_MONGODB_URI"] +async fn live_stream_disable_closes_and_enable_replaces() { + let engine = engine().await; + let desc = stream_table(&engine, "lifecycle").await; + let old = desc.latest_stream_arn.expect("arn"); + engine + .update_table( + ACCOUNT, + set_stream("lifecycle", serde_json::json!({"StreamEnabled": false})), + ) + .await + .expect("disable"); + let closed = describe_stream(&engine, &old).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::KeysOnly); + assert!( + closed + .shards + .iter() + .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) + ); + let re = engine + .update_table( + ACCOUNT, + set_stream( + "lifecycle", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ), + ) + .await + .expect("enable"); + let new = re.latest_stream_arn.expect("new arn"); + assert_ne!(new, old); + let d = describe_stream(&engine, &new).await; + assert!(matches!(d.stream_status, StreamStatus::Enabled)); + assert_eq!(d.stream_view_type, StreamViewType::NewImage); + audit(&engine, "lifecycle").await; + finish(engine).await; +} diff --git a/crates/storage-mongodb/src/stream_records_live_tests.rs b/crates/storage-mongodb/src/stream_records_live_tests.rs index bf769836a..3e5bb0a4c 100644 --- a/crates/storage-mongodb/src/stream_records_live_tests.rs +++ b/crates/storage-mongodb/src/stream_records_live_tests.rs @@ -34,6 +34,14 @@ fn uri() -> String { ) } +/// A shard id in the shape releases before stream labels were part of the id +/// wrote: shard `index` of table `table_id`. Such an id belongs to the +/// table's current stream, so these fixtures read as a table whose stream was +/// enabled by an earlier release. +fn legacy_shard_id(table_id: &str, index: u32) -> String { + format!("shardId-{table_id}-{index:012}") +} + struct Fixture { engine: MongoEngine, } @@ -77,7 +85,7 @@ impl Fixture { .data_db .collection::("stream_shards") .insert_one(doc! { - "shard_id": super::build_shard_id(&table_id, i), + "shard_id": legacy_shard_id(&table_id, i), "table_id": &table_id, "starting_sequence_number": format!("{:021}", 0), }) @@ -164,7 +172,7 @@ async fn records_live_a_page_over_16_mb_is_read_in_full() { let f = Fixture::new().await; let table_id = f.stream_table(ACCOUNT, "big", LABEL).await; let arn = stream_arn(REGION, ACCOUNT, "big", LABEL); - let shard_id = super::build_shard_id(&table_id, 0); + let shard_id = legacy_shard_id(&table_id, 0); // Inserted in descending order so the read's sort, not insertion order, // produces the sequence order the page is asserted in. let seqs: Vec = (1..=40).rev().collect(); @@ -209,7 +217,7 @@ async fn records_live_a_page_over_16_mb_is_read_in_full() { async fn records_live_an_earlier_label_on_the_same_shards_is_not_found() { let f = Fixture::new().await; let table_id = f.stream_table(ACCOUNT, "t", LABEL).await; - let shard_id = super::build_shard_id(&table_id, 0); + let shard_id = legacy_shard_id(&table_id, 0); f.write_records(&shard_id, &table_id, &[1], 10).await; let old_arn = stream_arn(REGION, ACCOUNT, "t", LABEL); f.engine @@ -243,8 +251,8 @@ async fn records_live_another_accounts_same_named_stream_is_not_found() { let f = Fixture::new().await; let own_id = f.stream_table(ACCOUNT, "t", LABEL).await; let other_id = f.stream_table(OTHER_ACCOUNT, "t", LABEL).await; - let own_shard = super::build_shard_id(&own_id, 0); - let other_shard = super::build_shard_id(&other_id, 0); + let own_shard = legacy_shard_id(&own_id, 0); + let other_shard = legacy_shard_id(&other_id, 0); f.write_records(&own_shard, &own_id, &[1], 10).await; f.write_records(&other_shard, &other_id, &[1], 10).await; @@ -285,9 +293,9 @@ async fn records_live_another_tables_shard_with_the_same_label_is_not_found() { let f = Fixture::new().await; let a_id = f.stream_table(ACCOUNT, "a", LABEL).await; let b_id = f.stream_table(ACCOUNT, "b", LABEL).await; - f.write_records(&super::build_shard_id(&a_id, 0), &a_id, &[1], 10) + f.write_records(&legacy_shard_id(&a_id, 0), &a_id, &[1], 10) .await; - f.write_records(&super::build_shard_id(&b_id, 0), &b_id, &[1, 2], 10) + f.write_records(&legacy_shard_id(&b_id, 0), &b_id, &[1, 2], 10) .await; let arn_a = stream_arn(REGION, ACCOUNT, "a", LABEL); let arn_b = stream_arn(REGION, ACCOUNT, "b", LABEL); @@ -296,12 +304,12 @@ async fn records_live_another_tables_shard_with_the_same_label_is_not_found() { // pick, one of them reads a shard that is not its own. assert_stream_not_found( f.engine - .get_stream_records(ACCOUNT, &arn_a, &super::build_shard_id(&b_id, 0), None, 10) + .get_stream_records(ACCOUNT, &arn_a, &legacy_shard_id(&b_id, 0), None, 10) .await, ); assert_stream_not_found( f.engine - .get_stream_records(ACCOUNT, &arn_b, &super::build_shard_id(&a_id, 0), None, 10) + .get_stream_records(ACCOUNT, &arn_b, &legacy_shard_id(&a_id, 0), None, 10) .await, ); assert_stream_not_found( @@ -311,7 +319,7 @@ async fn records_live_another_tables_shard_with_the_same_label_is_not_found() { ); let (records, _) = f .engine - .get_stream_records(ACCOUNT, &arn_a, &super::build_shard_id(&a_id, 0), None, 10) + .get_stream_records(ACCOUNT, &arn_a, &legacy_shard_id(&a_id, 0), None, 10) .await .expect("own shard reads"); assert_eq!(records.len(), 1); @@ -327,7 +335,7 @@ async fn records_live_a_record_document_without_record_data_is_an_internal_error let f = Fixture::new().await; let table_id = f.stream_table(ACCOUNT, "t", LABEL).await; let arn = stream_arn(REGION, ACCOUNT, "t", LABEL); - let shard_id = super::build_shard_id(&table_id, 0); + let shard_id = legacy_shard_id(&table_id, 0); f.write_records(&shard_id, &table_id, &[1], 10).await; f.engine .data_db @@ -369,8 +377,8 @@ async fn records_live_an_idle_shard_reads_an_empty_page() { let f = Fixture::new().await; let table_id = f.stream_table(ACCOUNT, "t", LABEL).await; let arn = stream_arn(REGION, ACCOUNT, "t", LABEL); - let written = super::build_shard_id(&table_id, 0); - let idle = super::build_shard_id(&table_id, 1); + let written = legacy_shard_id(&table_id, 0); + let idle = legacy_shard_id(&table_id, 1); f.write_records(&written, &table_id, &[1, 2], 10).await; let last_written = format!("{:021}", 2); diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 77706c3c7..6ca721b7d 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -149,6 +149,7 @@ impl MongoEngine { restore: Option<&crate::backup_engine::RestoreProvenance<'_>>, ) -> Result { Self::validate_account_id(account_id)?; + let input = extenddb_storage::stream_lineage::without_disabled_specification(input); let table_id = uuid::Uuid::new_v4().to_string(); let table_arn_val = table_arn(&self.region, account_id, &input.table_name); @@ -323,10 +324,16 @@ impl MongoEngine { } // Initialize stream shards if streaming is enabled. shard_id is - // derived from table_id (UUID), never table_name — see - // stream_engine::build_shard_id for the security rationale. - if stream_label_opt.is_some() { - self.init_stream_shards(&table_id).await?; + // derived from table_id (UUID), never table_name, and names the + // stream's label and view type (`extenddb_storage::stream_lineage`). + if let (Some(label), Some(view_type)) = ( + stream_label_opt.as_deref(), + input + .stream_specification + .as_ref() + .and_then(|s| s.stream_view_type), + ) { + self.init_stream_shards(&table_id, label, view_type).await?; } // Handle GSI creation @@ -816,68 +823,72 @@ impl MongoEngine { update_doc.insert("on_demand_throughput", odt_bson); } + // A stream change is decided on the specification and label the table + // document carries, and written only if they are still those values: + // this backend has no transaction around UpdateTable, so of two + // overlapping stream changes the one whose write finds the document + // changed re-reads it and is refused by `stream_change` as a request + // arriving after the other's commit would be. + let mut pending_stream = None; + let mut stream_guard = Document::new(); if let Some(ss) = &input.stream_specification { - let ss_bson = bson::to_bson(ss).map_err(|e| StorageError::Internal(e.to_string()))?; - update_doc.insert("stream_specification", ss_bson); - if ss.stream_enabled { - let table_doc = tables_coll - .find_one(doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; - let table_id = table_doc - .get_str("table_id") - .map_err(|_| StorageError::Internal("missing table_id".to_string()))?; - - // Idempotent re-enable: if shards already exist for this - // table, reuse them and preserve the existing - // stream_label. Otherwise a repeat UpdateTable would - // insert duplicate shards (DescribeStream would then - // report N × k) and rotate stream_label, invalidating - // stream ARNs previously handed out to consumers. - let shards_coll = self.data_db.collection::("stream_shards"); - let existing_shard = shards_coll - .find_one(doc! { "table_id": table_id }) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if existing_shard.is_none() { - let label = extenddb_storage::util::format_stream_label( - time::OffsetDateTime::now_utc(), - ); - update_doc.insert("stream_label", &label); - self.init_stream_shards(table_id).await?; - } else if table_doc - .get_str("stream_label") - .ok() - .filter(|s| !s.is_empty()) - .is_none() - { - // Shards exist but the label was cleared by a - // previous disable — restore a fresh label so the - // ARN resolves again. - let label = extenddb_storage::util::format_stream_label( - time::OffsetDateTime::now_utc(), - ); - update_doc.insert("stream_label", &label); - } + let table_doc = tables_coll + .find_one( + doc! { "_id": { "account_id": account_id, "table_name": &input.table_name } }, + ) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; + pending_stream = Some(Self::prepare_stream_update( + &mut update_doc, + &table_doc, + &input.table_name, + ss, + )?); + for field in ["stream_specification", "stream_label"] { + stream_guard.insert( + field, + table_doc.get(field).cloned().unwrap_or(bson::Bson::Null), + ); } } if !update_doc.is_empty() { + let mut filter = active_table_filter(&table_filter); + filter.extend(stream_guard); let result = tables_coll - .update_one( - active_table_filter(&table_filter), - doc! { "$set": &update_doc }, - ) + .update_one(filter, doc! { "$set": &update_doc }) .await .map_err(|e| StorageError::Internal(e.to_string()))?; if result.matched_count == 0 { - // Deleted or left ACTIVE since the check above; report which. + // Deleted or left ACTIVE since the check above, or the stream + // fields changed under a stream change; report which. A stream + // change the re-read state would still accept (a disable that + // overlapped an enable) is reported as not ACTIVE, the nearest + // of ExtendDB's answers to the service's in-progress refusal. self.require_active_table(&tables_coll, &table_filter, &input.table_name) .await?; + if let Some(ss) = &input.stream_specification { + let table_doc = tables_coll + .find_one(doc! { + "_id": { "account_id": account_id, "table_name": &input.table_name } + }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))? + .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; + let _would_apply = Self::prepare_stream_update( + &mut Document::new(), + &table_doc, + &input.table_name, + ss, + )?; + } return Err(StorageError::TableNotActive(input.table_name.clone())); } } + if let Some(pending) = pending_stream { + pending.apply(self, account_id, &input.table_name).await?; + } if clear_provisioned_throughput { let table_doc = tables_coll diff --git a/crates/storage-postgres/src/create_table.rs b/crates/storage-postgres/src/create_table.rs index fc41c8c0b..2d757ddd6 100755 --- a/crates/storage-postgres/src/create_table.rs +++ b/crates/storage-postgres/src/create_table.rs @@ -107,6 +107,7 @@ impl PostgresEngine { restore: Option<&RestoreRegistration<'_>>, ) -> Result { Self::validate_account_id(account_id)?; + let input = extenddb_storage::stream_lineage::without_disabled_specification(input); let table_id = uuid::Uuid::new_v4().to_string(); let table_arn = table_arn(&self.region, account_id, &input.table_name); let billing_mode = input.billing_mode.unwrap_or(BillingMode::Provisioned); @@ -437,22 +438,24 @@ impl PostgresEngine { // If data DDL fails, the catalog entry is cleaned up (see below). // Initialize stream shards and label if streams are enabled on this table. - let stream_label = if input + let stream_label = match input .stream_specification .as_ref() - .is_some_and(|s| s.stream_enabled) + .filter(|s| s.stream_enabled) + .and_then(|s| s.stream_view_type) { - let label = Self::init_stream_shards( - &mut tx, - &self.data_pool, - account_id, - &input.table_name, - &table_id, - ) - .await?; - Some(label) - } else { - None + Some(view_type) => Some( + Self::init_stream_shards( + &mut tx, + &self.data_pool, + account_id, + &input.table_name, + &table_id, + view_type, + ) + .await?, + ), + None => None, }; tx.commit() diff --git a/crates/storage-postgres/src/data/tx_helpers.rs b/crates/storage-postgres/src/data/tx_helpers.rs index 6468f159b..a77de1cb6 100755 --- a/crates/storage-postgres/src/data/tx_helpers.rs +++ b/crates/storage-postgres/src/data/tx_helpers.rs @@ -317,10 +317,18 @@ pub(super) async fn write_stream_record_in_tx( }) .unwrap_or_default(); + // Only the open shards: they belong to the table's enabled stream. A + // disabled stream's shards are closed and take no new records, so a write + // that read the table's key info before a disable records nothing. + // + // FOR SHARE: a disable locks the same rows FOR UPDATE before it takes the + // ending numbers, so this write either commits before the ending number + // is taken or, waiting here, re-evaluates `ending_sequence_number IS NULL` + // on the closed rows and records nothing (`close_stream_shards`). let shards: Vec<(String,)> = sqlx::query_as( "SELECT shard_id FROM stream_shards \ - WHERE table_id = $1 \ - ORDER BY shard_id", + WHERE table_id = $1 AND ending_sequence_number IS NULL \ + ORDER BY shard_id FOR SHARE", ) .bind(&key_info.table_id) .fetch_all(&mut **tx) diff --git a/crates/storage-postgres/src/lib.rs b/crates/storage-postgres/src/lib.rs index 9603d68ff..444e7a6a9 100755 --- a/crates/storage-postgres/src/lib.rs +++ b/crates/storage-postgres/src/lib.rs @@ -28,6 +28,7 @@ mod snapshot_engine; #[doc(hidden)] pub use snapshot_engine::map_fetch_error; mod stream_engine; +mod stream_lifecycle; #[doc(hidden)] pub mod stream_records_sql; mod table_engine; diff --git a/crates/storage-postgres/src/stream_engine.rs b/crates/storage-postgres/src/stream_engine.rs index 970a81755..9e5419d52 100755 --- a/crates/storage-postgres/src/stream_engine.rs +++ b/crates/storage-postgres/src/stream_engine.rs @@ -3,32 +3,17 @@ //! `StreamEngine` trait implementation for `PostgresEngine`. -use extenddb_core::types::{ - SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, StreamSummary, - StreamViewType, -}; +use extenddb_core::types::{StreamDescription, StreamRecord, StreamSummary}; use extenddb_storage::StreamEngine; use extenddb_storage::error::StorageError; -use extenddb_storage::util::{new_stream_label, parse_stream_arn, shard_in_stream, stream_arn}; +use extenddb_storage::stream_lineage; +use extenddb_storage::util::{parse_stream_arn, shard_in_stream, stream_arn}; use futures::future::BoxFuture; -use sqlx::PgPool; use crate::PostgresEngine; use crate::stream_records_sql::{STREAM_RECORDS_AFTER, STREAM_RECORDS_FROM_START}; -/// Number of fixed shards per stream (hash-based assignment). -const SHARDS_PER_STREAM: u32 = 4; - impl PostgresEngine { - /// Initialize stream shards for a table and set the `stream_label`. - /// - /// Updates the stream label in the catalog (via the provided catalog - /// transaction) and creates shard rows in the data database within a - /// data transaction for atomicity. - /// - /// # Errors - /// - /// Returns [`StorageError::Internal`] if any query fails. /// Remove the shards of deleted tables once their stream has aged out. /// /// DeleteTable leaves a table's shards and records in place, as the service @@ -92,64 +77,6 @@ impl PostgresEngine { } Ok(result.rows_affected()) } - - pub(crate) async fn init_stream_shards( - tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, - data_pool: &PgPool, - account_id: &str, - table_name: &str, - table_id: &str, - ) -> Result { - // Formatted here, not in SQL, so every backend issues the same label - // shape from one function (`extenddb_storage::util::format_stream_label`). - let label = new_stream_label(); - sqlx::query( - "UPDATE tables SET stream_label = $3 WHERE account_id = $1 AND table_name = $2", - ) - .bind(account_id) - .bind(table_name) - .bind(&label) - .execute(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - // P54 Bug 1: Stream shards live in the data database for atomic - // writes with stream records and item data. Use a transaction so - // all shards are created atomically. - let mut data_tx = data_pool - .begin() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - for i in 0..SHARDS_PER_STREAM { - // Derived from the table id, not its name. A name is reused by - // delete-and-recreate and by every account that picks it, while - // `stream_shards.shard_id` is unique across the data database - // and PostgreSQL keeps a deleted table's shards. A name of - // up to 255 bytes also overflowed the 65-character ShardId limit - // the AWS SDKs enforce. The table id is a UUID, giving a - // 61-character id. - let shard_id = format!("shardId-{table_id}-{i:016}"); - let start_seq = format!("{:021}", 0); - sqlx::query( - "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ - VALUES ($1, $2, $3)", - ) - .bind(&shard_id) - .bind(table_id) - .bind(&start_seq) - .execute(&mut *data_tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - - data_tx - .commit() - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - Ok(label) - } } impl StreamEngine for PostgresEngine { @@ -288,99 +215,24 @@ impl StreamEngine for PostgresEngine { let exclusive_start_shard_id = input.exclusive_start_shard_id.clone(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - - let row: Option<(serde_json::Value, serde_json::Value, Option, String, String)> = - sqlx::query_as( - "SELECT key_schema, attribute_definitions, stream_specification, table_status, table_id \ - FROM tables WHERE account_id = $1 AND table_name = $2 AND stream_label = $3", - ) - .bind(&account_id) - .bind(&table_name) - .bind(&stream_label) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let (ks_json, _ad_json, stream_spec_json, table_status, table_id) = - row.ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn))?; - + let (table, ks_json) = self + .stream_table_row(&account_id, &table_name) + .await? + .ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn))?; let key_schema = serde_json::from_value(ks_json) .map_err(|e| StorageError::Internal(e.to_string()))?; - - let stream_view_type = stream_spec_json - .and_then(|v| { - v.get("StreamViewType") - .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) - }) - .unwrap_or(StreamViewType::KeysOnly); - - let limit = limit.unwrap_or(100); - let shard_rows: Vec<(String, Option, String, Option)> = if let Some( - ref start, - ) = - exclusive_start_shard_id - { - sqlx::query_as( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM stream_shards WHERE table_id = $1 AND shard_id > $2 \ - ORDER BY shard_id LIMIT $3", - ) - .bind(&table_id) - .bind(start) - .bind(limit + 1) - .fetch_all(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - } else { - sqlx::query_as( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM stream_shards WHERE table_id = $1 \ - ORDER BY shard_id LIMIT $2", - ) - .bind(&table_id) - .bind(limit + 1) - .fetch_all(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - }; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - let last_shard = if shard_rows.len() > limit_usize { - Some(shard_rows[limit_usize - 1].0.clone()) - } else { - None - }; - - let shards: Vec = shard_rows - .into_iter() - .take(limit_usize) - .map(|(id, parent, start, end)| Shard { - shard_id: id, - parent_shard_id: parent, - sequence_number_range: SequenceNumberRange { - starting_sequence_number: start, - ending_sequence_number: end, - }, - }) - .collect(); - - let stream_status = if table_status == "DELETING" { - StreamStatus::Disabling - } else { - StreamStatus::Enabled - }; - - Ok(StreamDescription { + let shards = self + .stream_shard_rows(std::slice::from_ref(&table.table_id)) + .await?; + stream_lineage::describe_stream( stream_arn, - stream_label, - stream_status, - stream_view_type, - table_name, + &table, key_schema, - shards, - last_evaluated_shard_id: last_shard, - }) + stream_label, + &shards, + limit, + exclusive_start_shard_id.as_deref(), + ) }) } @@ -396,84 +248,21 @@ impl StreamEngine for PostgresEngine { let exclusive_start_stream_arn = exclusive_start_stream_arn.map(std::string::ToString::to_string); Box::pin(async move { - let rows: Vec<(String, String, String)> = match ( + let start = exclusive_start_stream_arn + .as_deref() + .map(parse_stream_arn) + .transpose()?; + let tables = self.stream_table_rows(&account_id).await?; + let ids: Vec = tables.iter().map(|t| t.table_id.clone()).collect(); + let shards = self.stream_shard_rows(&ids).await?; + stream_lineage::list_streams( + &tables, + &shards, table_name.as_deref(), - exclusive_start_stream_arn.as_deref(), - ) { - (Some(tn), Some(start_arn)) => { - let (_, start_label) = parse_stream_arn(start_arn)?; - sqlx::query_as( - "SELECT table_name, table_arn, stream_label FROM tables \ - WHERE account_id = $1 AND stream_label IS NOT NULL AND table_name = $2 AND stream_label > $3 \ - ORDER BY stream_label LIMIT $4", - ) - .bind(&account_id) - .bind(tn) - .bind(&start_label) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - } - (Some(tn), None) => sqlx::query_as( - "SELECT table_name, table_arn, stream_label FROM tables \ - WHERE account_id = $1 AND stream_label IS NOT NULL AND table_name = $2 \ - ORDER BY stream_label LIMIT $3", - ) - .bind(&account_id) - .bind(tn) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?, - (None, Some(start_arn)) => { - let (start_table, start_label) = parse_stream_arn(start_arn)?; - sqlx::query_as( - "SELECT table_name, table_arn, stream_label FROM tables \ - WHERE account_id = $1 AND stream_label IS NOT NULL \ - AND (table_name, stream_label) > ($2, $3) \ - ORDER BY table_name, stream_label LIMIT $4", - ) - .bind(&account_id) - .bind(&start_table) - .bind(&start_label) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))? - } - (None, None) => sqlx::query_as( - "SELECT table_name, table_arn, stream_label FROM tables \ - WHERE account_id = $1 AND stream_label IS NOT NULL \ - ORDER BY table_name, stream_label LIMIT $2", - ) - .bind(&account_id) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?, - }; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit as usize; - - let summaries: Vec = rows - .iter() - .take(limit_usize) - .map(|(tn, _table_arn, label)| StreamSummary { - stream_arn: stream_arn(&self.region, &account_id, tn, label), - stream_label: label.clone(), - table_name: tn.clone(), - }) - .collect(); - - let last_arn = if rows.len() > limit_usize { - summaries.last().map(|s| s.stream_arn.clone()) - } else { - None - }; - - Ok((summaries, last_arn)) + limit, + start, + |t, l| stream_arn(&self.region, &account_id, t, l), + ) }) } @@ -514,9 +303,11 @@ impl StreamEngine for PostgresEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Only the open shards: they belong to the table's enabled stream. + // A disabled stream's closed shards take no new records. let shards: Vec<(String,)> = sqlx::query_as( "SELECT shard_id FROM stream_shards \ - WHERE table_id = $1 \ + WHERE table_id = $1 AND ending_sequence_number IS NULL \ ORDER BY shard_id", ) .bind(&table_id) @@ -558,34 +349,20 @@ impl StreamEngine for PostgresEngine { let shard_id = shard_id.to_string(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - - let table_id: Option = sqlx::query_scalar( - "SELECT table_id FROM tables \ - WHERE account_id = $1 AND table_name = $2 AND stream_label = $3", - ) - .bind(&account_id) - .bind(&table_name) - .bind(&stream_label) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let Some(table_id) = table_id else { - return Err(extenddb_storage::util::stream_does_not_exist()); + let table = self.stream_table_row(&account_id, &table_name).await?; + let shards = match &table { + Some((t, _)) => { + self.stream_shard_rows(std::slice::from_ref(&t.table_id)) + .await? + } + None => Vec::new(), }; - - let exists: Option<(i32,)> = - sqlx::query_as("SELECT 1 FROM stream_shards WHERE shard_id = $1 AND table_id = $2") - .bind(&shard_id) - .bind(&table_id) - .fetch_optional(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if exists.is_none() { - return Err(extenddb_storage::util::shard_not_in_stream(&stream_arn)); - } - Ok(()) + stream_lineage::validate_shard( + table.as_ref().map(|(t, _)| t), + &stream_label, + &shards, + &shard_id, + ) }) } diff --git a/crates/storage-postgres/src/stream_lifecycle.rs b/crates/storage-postgres/src/stream_lifecycle.rs new file mode 100644 index 000000000..b5b038672 --- /dev/null +++ b/crates/storage-postgres/src/stream_lifecycle.rs @@ -0,0 +1,381 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stream lifecycle on PostgreSQL: creating a table's stream, closing it on +//! `UpdateTable StreamEnabled=false`, and reading the table and shard rows the +//! shared rules in `extenddb_storage::stream_lineage` group into streams. +//! +//! The table row is in the catalog database and the shards and records in the +//! data database, so a stream change is two commits. `UpdateTable` changes the +//! catalog row (specification and label) in its transaction and changes the +//! shards after that transaction commits ([`PendingStream`]). A failure between +//! the two leaves the catalog's answer in force: a disabled stream whose shards +//! are still open (closed by the next change of that stream), or an enabled stream with no +//! shards, which records nothing and is replaced by a disable and an enable. + +use extenddb_core::types::{StreamSpecification, StreamViewType}; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, enable_still_current, + next_stream_label, open_shard_ids_of, shard_id, stored_specification, stream_change, +}; +use sqlx::PgPool; + +use crate::PostgresEngine; + +type ShardTuple = (String, String, Option, String, Option); + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +fn spec_of(value: Option) -> Result, StorageError> { + value + .map(serde_json::from_value) + .transpose() + .map_err(internal) +} + +/// The shard changes of a committed `UpdateTable` stream change: close the +/// open shards of the stream the change read (`close`, the table's label at +/// that time), then, on an enable, create the new stream's shards. +#[must_use] +pub(crate) struct PendingStream { + table_id: String, + close: Option, + open: Option<(String, StreamViewType)>, +} + +impl PendingStream { + /// Run the shard changes once the catalog transaction has committed. + /// + /// The enable's shards are inserted after its catalog row is visible, so a + /// disable that reads the row in between passes its check, commits the + /// disabled specification and closes the shards it finds, which may be + /// none or some of the four. The enable re-reads the table's stream fields + /// after its inserts and, when the row no longer says enabled with this + /// label, closes the shards it inserted. A disable committed before the + /// re-read is seen by it; one committed after finds all four rows open + /// and closes them itself; both closing is idempotent through the + /// `ending_sequence_number IS NULL` guard. + pub(crate) async fn apply( + self, + engine: &PostgresEngine, + account_id: &str, + table_name: &str, + ) -> Result<(), StorageError> { + let data_pool = &engine.data_pool; + if let Some(label) = &self.close { + PostgresEngine::close_stream_shards(data_pool, &self.table_id, label).await?; + } + if let Some((label, view_type)) = self.open { + PostgresEngine::insert_stream_shards(data_pool, &self.table_id, &label, view_type) + .await?; + let row = engine.stream_table_row(account_id, table_name).await?; + if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { + PostgresEngine::close_stream_shards(data_pool, &self.table_id, &label).await?; + } + } + Ok(()) + } +} + +impl PostgresEngine { + /// Create a new stream on the table: `tables.stream_label` set to a new + /// label in the caller's catalog transaction, and four open shards named + /// for it committed in the data database. Returns the label. + pub(crate) async fn init_stream_shards( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + data_pool: &PgPool, + account_id: &str, + table_name: &str, + table_id: &str, + view_type: StreamViewType, + ) -> Result { + let label = Self::set_new_stream_label(tx, account_id, table_name).await?; + Self::insert_stream_shards(data_pool, table_id, &label, view_type).await?; + Ok(label) + } + + /// Set `tables.stream_label` to a label later than the table's previous + /// one, in the caller's catalog transaction. + async fn set_new_stream_label( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + account_id: &str, + table_name: &str, + ) -> Result { + let previous: Option = sqlx::query_scalar( + "SELECT stream_label FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let label = next_stream_label(previous.as_deref()); + sqlx::query( + "UPDATE tables SET stream_label = $3 WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .bind(&label) + .execute(&mut **tx) + .await + .map_err(internal)?; + Ok(label) + } + + /// Insert the four open shards of stream `label`. Stream shards live in + /// the data database for atomic writes with stream records and item data; + /// one transaction creates all four. + async fn insert_stream_shards( + data_pool: &PgPool, + table_id: &str, + label: &str, + view_type: StreamViewType, + ) -> Result<(), StorageError> { + let mut data_tx = data_pool.begin().await.map_err(internal)?; + for i in 0..SHARDS_PER_STREAM { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES ($1, $2, $3)", + ) + .bind(shard_id(table_id, label, view_type, i)?) + .bind(table_id) + .bind(format!("{:021}", 0)) + .execute(&mut *data_tx) + .await + .map_err(internal)?; + } + data_tx.commit().await.map_err(internal)?; + Ok(()) + } + + /// Close every open shard of the table: its `EndingSequenceNumber` becomes + /// the sequence number of its last record, or its starting number when it + /// holds none (both measured 2026-10-24). + /// + /// The service's rule, measured 2026-10-24 with 569 writes racing a + /// disable: every write accepted before the stream reports `DISABLED` + /// lands in the closed stream before its shard's `EndingSequenceNumber`, + /// and a write after that records nothing. Held here by a lock in the data + /// database: the write path reads the open shards `FOR SHARE` in its data + /// transaction, and this close locks them `FOR UPDATE` first and takes the + /// ending numbers in the same transaction, so a write that read the open + /// set commits before the ending number is taken, and a write that arrives + /// after the lock re-evaluates the closed rows and records nothing. What + /// remains is the gap every backend has: a write acknowledged after the + /// disable response that recorded into no stream, because its key info + /// pre-dated the disable and it found no open shard. A `TransactWriteItems` + /// across two tables whose disables run at the same instant can form a + /// lock cycle through the shard locks; PostgreSQL's deadlock detector + /// cancels one side, which the transaction path reports as a cancellation. + pub(crate) async fn close_stream_shards( + data_pool: &PgPool, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + let rows: Vec = sqlx::query_as( + "SELECT table_id, shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number \ + FROM stream_shards WHERE table_id = $1", + ) + .bind(table_id) + .fetch_all(data_pool) + .await + .map_err(internal)?; + let ids = open_shard_ids_of(table_id, label, &shard_rows(rows)); + if ids.is_empty() { + return Ok(()); + } + let mut data_tx = data_pool.begin().await.map_err(internal)?; + sqlx::query( + "SELECT shard_id FROM stream_shards \ + WHERE shard_id = ANY($1) AND ending_sequence_number IS NULL FOR UPDATE", + ) + .bind(&ids) + .fetch_all(&mut *data_tx) + .await + .map_err(internal)?; + sqlx::query( + "UPDATE stream_shards s SET ending_sequence_number = COALESCE(\ + (SELECT MAX(r.sequence_number) FROM stream_records r \ + WHERE r.shard_id = s.shard_id), \ + s.starting_sequence_number) \ + WHERE s.shard_id = ANY($1) AND s.ending_sequence_number IS NULL", + ) + .bind(&ids) + .execute(&mut *data_tx) + .await + .map_err(internal)?; + data_tx.commit().await.map_err(internal)?; + Ok(()) + } + + /// Apply an `UpdateTable` `StreamSpecification` in the caller's catalog + /// transaction, which holds the table row `FOR UPDATE`. + /// + /// Disabling clears the stored specification, so DescribeTable omits + /// `StreamSpecification`, and keeps `stream_label`, so `LatestStreamArn` + /// keeps naming the disabled stream. Enabling stores the specification and + /// a new label. The shard changes are returned, for the caller to apply + /// with [`PendingStream::apply`] once the catalog has committed: a request + /// refused after this point (an invalid index update in the same call) + /// then leaves no shards behind. + pub(crate) async fn apply_stream_specification( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + account_id: &str, + table_name: &str, + table_id: &str, + requested: &StreamSpecification, + ) -> Result { + let current: Option = sqlx::query_scalar( + "SELECT stream_specification FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let current = spec_of(current)?; + let current_label: Option = sqlx::query_scalar( + "SELECT stream_label FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let change = stream_change(table_name, current.as_ref(), requested)?; + let stored = + serde_json::to_value(stored_specification(change, current.as_ref(), requested)) + .map_err(internal)?; + sqlx::query( + "UPDATE tables SET stream_specification = $1 \ + WHERE account_id = $2 AND table_name = $3", + ) + .bind(stored) + .bind(account_id) + .bind(table_name) + .execute(&mut **tx) + .await + .map_err(internal)?; + let open = match change { + StreamChange::Disable => None, + StreamChange::Enable(view_type) => Some(( + Self::set_new_stream_label(tx, account_id, table_name).await?, + view_type, + )), + }; + Ok(PendingStream { + table_id: table_id.to_owned(), + close: current_label, + open, + }) + } + + /// The table's stream columns and its key schema, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + type Row = ( + String, + String, + Option, + Option, + serde_json::Value, + ); + let row: Option = sqlx::query_as( + "SELECT table_id, table_status, stream_label, stream_specification, key_schema \ + FROM tables WHERE account_id = $1 AND table_name = $2", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(internal)?; + row.map(|(table_id, table_status, stream_label, spec, key_schema)| { + Ok(( + TableStreamRow { + table_name: table_name.to_owned(), + table_id, + table_status, + stream_label, + stream_specification: spec_of(spec)?, + }, + key_schema, + )) + }) + .transpose() + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + type Row = ( + String, + String, + String, + Option, + Option, + ); + let rows: Vec = sqlx::query_as( + "SELECT table_name, table_id, table_status, stream_label, stream_specification \ + FROM tables WHERE account_id = $1 AND stream_label IS NOT NULL", + ) + .bind(account_id) + .fetch_all(&self.pool) + .await + .map_err(internal)?; + rows.into_iter() + .map(|(table_name, table_id, table_status, stream_label, spec)| { + Ok(TableStreamRow { + table_name, + table_id, + table_status, + stream_label, + stream_specification: spec_of(spec)?, + }) + }) + .collect() + } + + /// The shard rows of the given tables. + pub(crate) async fn stream_shard_rows( + &self, + table_ids: &[String], + ) -> Result, StorageError> { + if table_ids.is_empty() { + return Ok(Vec::new()); + } + let rows: Vec = sqlx::query_as( + "SELECT table_id, shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number \ + FROM stream_shards WHERE table_id = ANY($1)", + ) + .bind(table_ids) + .fetch_all(&self.data_pool) + .await + .map_err(internal)?; + Ok(shard_rows(rows)) + } +} + +fn shard_rows(rows: Vec) -> Vec { + rows.into_iter() + .map(|(table_id, shard_id, parent, start, end)| ShardRow { + table_id, + shard_id, + parent_shard_id: parent, + starting_sequence_number: start, + ending_sequence_number: end, + }) + .collect() +} diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index dec99a93b..61b4174ef 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -331,75 +331,21 @@ impl PostgresEngine { .map_err(|e| StorageError::Internal(e.to_string()))?; } - // Apply stream specification change (enable/disable streams). - if let Some(spec) = &input.stream_specification { - let spec_json = - serde_json::to_value(spec).map_err(|e| StorageError::Internal(e.to_string()))?; - sqlx::query( - "UPDATE tables SET stream_specification = $1 \ - WHERE account_id = $2 AND table_name = $3", - ) - .bind(&spec_json) - .bind(account_id) - .bind(&input.table_name) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if spec.stream_enabled { - // Check if shards already exist (re-enabling streams on a table - // that previously had them). Query the data pool since - // stream_shards lives in the data database. - let existing: Option<(String,)> = sqlx::query_as( - "SELECT shard_id FROM stream_shards \ - WHERE table_id = $1 \ - LIMIT 1", + // Apply stream specification change (enable/disable streams). The + // shard changes run once the catalog has committed. + let pending_stream = match &input.stream_specification { + Some(spec) => Some( + Self::apply_stream_specification( + &mut tx, + account_id, + &input.table_name, + &table_id, + spec, ) - .bind(&table_id) - .fetch_optional(&self.data_pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if existing.is_none() { - Self::init_stream_shards( - &mut tx, - &self.data_pool, - account_id, - &input.table_name, - &table_id, - ) - .await?; - } else { - // Shards exist but stream_label may be NULL if streams were - // previously disabled and the disable path cleared the label. - // This is a defensive check — init_stream_shards sets the - // label on first enable, but re-enable after disable needs - // to restore it. - let current_label: Option = sqlx::query_scalar( - "SELECT stream_label FROM tables \ - WHERE account_id = $1 AND table_name = $2", - ) - .bind(account_id) - .bind(&input.table_name) - .fetch_one(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - if current_label.is_none() { - sqlx::query( - "UPDATE tables SET stream_label = $3 \ - WHERE account_id = $1 AND table_name = $2", - ) - .bind(account_id) - .bind(&input.table_name) - .bind(extenddb_storage::util::new_stream_label()) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - } - } - } + .await?, + ), + None => None, + }; // Apply table class change. if let Some(tc) = &input.table_class { @@ -901,6 +847,10 @@ impl PostgresEngine { return Err(e); } + if let Some(pending) = pending_stream { + pending.apply(self, account_id, &input.table_name).await?; + } + // Drop the data tables of deleted GSIs, after the catalog commit. if let Some(updates) = &input.global_secondary_index_updates { let mut delete_idx = 0usize; diff --git a/crates/storage-postgres/tests/stream_disable.rs b/crates/storage-postgres/tests/stream_disable.rs new file mode 100644 index 000000000..41fe212c5 --- /dev/null +++ b/crates/storage-postgres/tests/stream_disable.rs @@ -0,0 +1,544 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Disabling a table's stream closes its shards, and enabling it again +//! creates a new stream, against the rows measured on the service on +//! 2026-10-24. +//! +//! Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (host-only, e.g. +//! `postgres://user:pass@127.0.0.1:5432`); skips otherwise. Each test builds +//! its own scratch database and drops it afterwards. + +use extenddb_core::expression::ExpressionMaps; +use extenddb_core::types::{ + AttributeDefinition, BillingMode, CreateTableInput, DescribeStreamInput, DescribeTableInput, + KeySchemaElement, KeyType, ScalarAttributeType, StreamDescription, StreamSpecification, + StreamStatus, StreamViewType, TableDescription, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, TableEngine}; +use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; +use sqlx::PgPool; +use sqlx::postgres::PgPoolOptions; + +const ACCOUNT: &str = "123456789012"; +const REGION: &str = "us-east-1"; + +struct Scratch { + engine: PostgresEngine, + db: PgPool, + admin: PgPool, + db_name: String, +} + +impl Scratch { + async fn cleanup(self) { + let Scratch { + engine, + db, + admin, + db_name, + } = self; + drop(engine); + db.close().await; + sqlx::query(&format!( + "DROP DATABASE IF EXISTS \"{db_name}\" WITH (FORCE)" + )) + .execute(&admin) + .await + .expect("drop the scratch database"); + admin.close().await; + } +} + +fn base_conn() -> Option { + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; + (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) +} + +/// Every `.sql` file under `migrations/` and `data_migrations/`, each +/// directory in filename order. +fn shipped_migrations() -> Vec { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut out = Vec::new(); + for dir in ["migrations", "data_migrations"] { + let mut files: Vec<_> = std::fs::read_dir(root.join(dir)) + .expect("read the migrations directory") + .map(|e| e.expect("directory entry").path()) + .filter(|p| p.extension().is_some_and(|e| e == "sql")) + .collect(); + files.sort(); + for f in files { + out.push(std::fs::read_to_string(&f).expect("read a migration file")); + } + } + out +} + +async fn scratch() -> Scratch { + let base = base_conn().expect("caller checks base_conn() first"); + let db_name = format!("eddb_sdis_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + .expect("connect to the postgres maintenance database"); + sqlx::query(&format!("CREATE DATABASE \"{db_name}\"")) + .execute(&admin) + .await + .expect("create the scratch database"); + let url = format!("{base}/{db_name}"); + let db = PgPoolOptions::new() + .max_connections(2) + .connect(&url) + .await + .expect("connect to the scratch database"); + for sql in shipped_migrations() { + sqlx::raw_sql(&sql) + .execute(&db) + .await + .expect("apply a shipped migration"); + } + // One scratch database stands in for both the catalog and the data + // database; drop the foreign key that cannot exist across the two. + sqlx::query("ALTER TABLE stream_shards DROP CONSTRAINT IF EXISTS stream_shards_table_id_fkey") + .execute(&db) + .await + .expect("match the two-database layout"); + sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") + .bind(ACCOUNT) + .bind(format!("acct-{db_name}")) + .execute(&db) + .await + .expect("seed the account row"); + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: url, + pool_size: 10, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("open a PostgresEngine on the scratch database"); + Scratch { + engine, + db, + admin, + db_name, + } +} + +fn table_input(name: &str, stream: bool) -> CreateTableInput { + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + billing_mode: Some(BillingMode::PayPerRequest), + stream_specification: stream.then_some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }), + ..Default::default() + } +} + +/// Finish the control-plane transition the worker would run. +async fn activate(s: &Scratch, name: &str) { + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_name = $1") + .bind(name) + .execute(&s.db) + .await + .expect("activate"); +} + +async fn set_stream( + s: &Scratch, + name: &str, + spec: serde_json::Value, +) -> Result { + let input = serde_json::json!({"TableName": name, "StreamSpecification": spec}); + let out = s + .engine + .update_table(ACCOUNT, serde_json::from_value(input).expect("input")) + .await; + activate(s, name).await; + out +} + +async fn describe_table(s: &Scratch, name: &str) -> TableDescription { + s.engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .expect("describe table") +} + +async fn describe_stream(s: &Scratch, arn: &str) -> StreamDescription { + s.engine + .describe_stream( + ACCOUNT, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .expect("describe stream") +} + +async fn put(s: &Scratch, name: &str, pk: &str, view_type: StreamViewType) { + let key_info = s + .engine + .table_key_info(ACCOUNT, name) + .await + .expect("key info"); + let item = serde_json::from_value(serde_json::json!({"pk": {"S": pk}})).expect("item"); + let capture = StreamCapture { + view_type, + user_identity: None, + region: REGION.into(), + }; + s.engine + .put_item( + &key_info, + item, + false, + None, + &ExpressionMaps::default(), + Some(&capture), + ) + .await + .expect("put"); +} + +async fn last_record(s: &Scratch, shard: &str) -> Option { + sqlx::query_scalar("SELECT MAX(sequence_number) FROM stream_records WHERE shard_id = $1") + .bind(shard) + .fetch_one(&s.db) + .await + .expect("last record") +} + +async fn record_count(s: &Scratch) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM stream_records") + .fetch_one(&s.db) + .await + .expect("count records") +} + +#[tokio::test] +async fn disable_closes_every_shard_and_re_enable_creates_a_new_stream() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("t482", true)) + .await + .expect("create"); + activate(&s, "t482").await; + for pk in ["k0", "k1", "k2"] { + put(&s, "t482", pk, StreamViewType::NewAndOldImages).await; + } + let enabled = describe_table(&s, "t482").await; + let old_arn = enabled.latest_stream_arn.clone().expect("stream arn"); + + let disabled = set_stream(&s, "t482", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let mut disabled = disabled; + disabled.prepare_for_response(); + assert!(disabled.stream_specification.is_none()); + assert_eq!( + disabled.latest_stream_arn.as_deref(), + Some(old_arn.as_str()) + ); + let mut table = describe_table(&s, "t482").await; + table.prepare_for_response(); + assert!(table.stream_specification.is_none()); + assert_eq!(table.latest_stream_label, enabled.latest_stream_label); + + let closed = describe_stream(&s, &old_arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::NewAndOldImages); + assert_eq!(closed.shards.len(), 4); + for shard in &closed.shards { + let range = &shard.sequence_number_range; + let expected = last_record(&s, &shard.shard_id) + .await + .unwrap_or_else(|| range.starting_sequence_number.clone()); + assert_eq!( + range.ending_sequence_number.as_deref(), + Some(expected.as_str()) + ); + } + + // A write still carrying the enabled view type records nothing. + put(&s, "t482", "stale", StreamViewType::NewAndOldImages).await; + assert_eq!(record_count(&s).await, 3); + + match set_stream(&s, "t482", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: t482"); + } + other => panic!("{other:?}"), + } + + let reenabled = set_stream( + &s, + "t482", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("re-enable"); + let new_arn = reenabled.latest_stream_arn.clone().expect("new arn"); + assert_ne!(new_arn, old_arn); + let new = describe_stream(&s, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.stream_view_type, StreamViewType::KeysOnly); + assert!( + new.shards + .iter() + .all(|n| !closed.shards.iter().any(|o| o.shard_id == n.shard_id)) + ); + match set_stream( + &s, + "t482", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ) + .await + { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table already has an enabled stream: TableName: t482"); + } + other => panic!("{other:?}"), + } + + put(&s, "t482", "after", StreamViewType::KeysOnly).await; + assert_eq!(record_count(&s).await, 4); + let old_again = describe_stream(&s, &old_arn).await; + assert!(matches!(old_again.stream_status, StreamStatus::Disabled)); + for (a, b) in old_again.shards.iter().zip(&closed.shards) { + assert_eq!(a.shard_id, b.shard_id); + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number + ); + } + + let (listed, _) = s + .engine + .list_streams(ACCOUNT, Some("t482"), 100, None) + .await + .expect("list"); + let arns: Vec<&str> = listed.iter().map(|x| x.stream_arn.as_str()).collect(); + assert_eq!(arns, [new_arn.as_str(), old_arn.as_str()]); + match s + .engine + .validate_shard(ACCOUNT, &old_arn, &new.shards[0].shard_id) + .await + { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Shard does not exist"); + } + other => panic!("{other:?}"), + } + s.cleanup().await; +} + +#[tokio::test] +async fn disable_on_a_table_that_never_had_a_stream_is_refused() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("nostream", false)) + .await + .expect("create"); + activate(&s, "nostream").await; + match set_stream(&s, "nostream", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: nostream"); + } + other => panic!("{other:?}"), + } + // An UpdateTable that enables a stream and is then refused for an index + // change in the same call leaves no shards and no label behind. + let input = serde_json::json!({ + "TableName": "nostream", + "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}, + "GlobalSecondaryIndexUpdates": [{"Delete": {"IndexName": "missing"}}], + }); + let refused = s + .engine + .update_table(ACCOUNT, serde_json::from_value(input).expect("input")) + .await; + assert!(refused.is_err(), "{refused:?}"); + let shards: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stream_shards") + .fetch_one(&s.db) + .await + .expect("count shards"); + assert_eq!(shards, 0); + assert!( + describe_table(&s, "nostream") + .await + .latest_stream_arn + .is_none() + ); + s.cleanup().await; +} + +/// A stream-enabled table created under v0.1.13 or earlier has four shard +/// rows named `shardId-{table_name}-{index:016}`. Its stream stays +/// describable, readable and listed after the upgrade, keeps its view type +/// once disabled, and is replaced by a labelled stream on re-enable. +#[tokio::test] +async fn a_stream_from_before_v0_1_14_survives_the_upgrade() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("legacy", false)) + .await + .expect("create"); + activate(&s, "legacy").await; + let table_id = describe_table(&s, "legacy").await.table_id; + // What a pre-v0.1.14 release wrote: the label and specification on the + // catalog row, four name-based open shards in the data database. + let label = "2026-10-20T10:00:00.000"; + sqlx::query( + "UPDATE tables SET stream_label = $1, stream_specification = $2 \ + WHERE account_id = $3 AND table_name = 'legacy'", + ) + .bind(label) + .bind(serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"})) + .bind(ACCOUNT) + .execute(&s.db) + .await + .expect("seed the catalog row"); + for i in 0..4 { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES ($1, $2, $3)", + ) + .bind(format!("shardId-legacy-{i:016}")) + .bind(&table_id) + .bind(format!("{:021}", 0)) + .execute(&s.db) + .await + .expect("seed a legacy shard"); + } + let arn = describe_table(&s, "legacy") + .await + .latest_stream_arn + .expect("the seeded stream arn"); + assert!(arn.ends_with(label)); + + put(&s, "legacy", "k", StreamViewType::NewImage).await; + let open = describe_stream(&s, &arn).await; + assert!(matches!(open.stream_status, StreamStatus::Enabled)); + assert_eq!(open.stream_view_type, StreamViewType::NewImage); + let ids: Vec<&str> = open.shards.iter().map(|x| x.shard_id.as_str()).collect(); + assert_eq!( + ids, + (0..4) + .map(|i| format!("shardId-legacy-{i:016}")) + .collect::>() + ); + s.engine + .validate_shard(ACCOUNT, &arn, "shardId-legacy-0000000000000001") + .await + .expect("a legacy shard belongs to the stream"); + assert_eq!(record_count(&s).await, 1); + let (listed, _) = s + .engine + .list_streams(ACCOUNT, Some("legacy"), 100, None) + .await + .expect("list"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].stream_arn, arn); + + set_stream(&s, "legacy", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let closed = describe_stream(&s, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!( + closed.stream_view_type, + StreamViewType::NewImage, + "a disabled legacy stream keeps its view type" + ); + assert!( + closed + .shards + .iter() + .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) + ); + // The backend keeps the disabled specification (with the view type) and + // the engine omits it from every response. + let stored = describe_table(&s, "legacy").await.stream_specification; + assert_eq!( + stored, + Some(StreamSpecification { + stream_enabled: false, + stream_view_type: Some(StreamViewType::NewImage), + }) + ); + let mut rendered = describe_table(&s, "legacy").await; + rendered.prepare_for_response(); + assert!(rendered.stream_specification.is_none()); + put(&s, "legacy", "dark", StreamViewType::NewImage).await; + assert_eq!( + record_count(&s).await, + 1, + "a write after the disable records nothing" + ); + + let re = set_stream( + &s, + "legacy", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("re-enable"); + let new_arn = re.latest_stream_arn.expect("new arn"); + assert_ne!(new_arn, arn); + let new = describe_stream(&s, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.shards.len(), 4); + assert!(new.shards.iter().all(|x| x.shard_id.len() == 61)); + // The enable closed the four legacy rows: none is still open. + let open_legacy: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_shards \ + WHERE shard_id LIKE 'shardId-legacy-%' AND ending_sequence_number IS NULL", + ) + .fetch_one(&s.db) + .await + .expect("count open legacy shards"); + assert_eq!( + open_legacy, 0, + "the enable closes the legacy stream's shards" + ); + s.cleanup().await; +} diff --git a/crates/storage-sqlite/src/create_table.rs b/crates/storage-sqlite/src/create_table.rs index ae4ae05e7..6a4d0f693 100644 --- a/crates/storage-sqlite/src/create_table.rs +++ b/crates/storage-sqlite/src/create_table.rs @@ -68,6 +68,7 @@ impl SqliteEngine { restore: Option<&RestoreRegistration<'_>>, ) -> Result { Self::validate_account_id(account_id)?; + let input = extenddb_storage::stream_lineage::without_disabled_specification(input); // D1: every writer holds the engine write lock. This method runs two // write transactions (catalog rows, then data-table DDL); without the // lock they contend with data-plane writers at the SQLite level, and a @@ -419,14 +420,23 @@ impl SqliteEngine { } // Stream shards (same transaction — single file). - let stream_label = if input + let stream_label = match input .stream_specification .as_ref() - .is_some_and(|s| s.stream_enabled) + .filter(|s| s.stream_enabled) + .and_then(|s| s.stream_view_type) { - Some(Self::init_stream_shards(&mut tx, account_id, &input.table_name, &table_id).await?) - } else { - None + Some(view_type) => Some( + Self::init_stream_shards( + &mut tx, + account_id, + &input.table_name, + &table_id, + view_type, + ) + .await?, + ), + None => None, }; tx.commit() diff --git a/crates/storage-sqlite/src/data/tx_helpers.rs b/crates/storage-sqlite/src/data/tx_helpers.rs index 1fd1bfca8..e28e71db9 100644 --- a/crates/storage-sqlite/src/data/tx_helpers.rs +++ b/crates/storage-sqlite/src/data/tx_helpers.rs @@ -217,12 +217,17 @@ pub(super) async fn write_stream_record_in_tx( }) .unwrap_or_default(); - let shards: Vec<(String,)> = - sqlx::query_as("SELECT shard_id FROM stream_shards WHERE table_id = ? ORDER BY shard_id") - .bind(&key_info.table_id) - .fetch_all(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; + // Only the open shards: they belong to the table's enabled stream. A + // disabled stream's shards are closed and take no new records, so a write + // that read the table's key info before a disable records nothing. + let shards: Vec<(String,)> = sqlx::query_as( + "SELECT shard_id FROM stream_shards \ + WHERE table_id = ? AND ending_sequence_number IS NULL ORDER BY shard_id", + ) + .bind(&key_info.table_id) + .fetch_all(&mut **tx) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; if shards.is_empty() { return Ok(()); // Streams not enabled for this table. } diff --git a/crates/storage-sqlite/src/lib.rs b/crates/storage-sqlite/src/lib.rs index 7e8085ff9..d428ad38e 100644 --- a/crates/storage-sqlite/src/lib.rs +++ b/crates/storage-sqlite/src/lib.rs @@ -41,6 +41,7 @@ mod snapshot_engine; mod sqlite_util; mod store; mod stream; +mod stream_lifecycle; #[cfg(test)] mod stream_tests; mod table_engine; diff --git a/crates/storage-sqlite/src/stream.rs b/crates/storage-sqlite/src/stream.rs index 7cd3d4729..bf2b8c2df 100644 --- a/crates/storage-sqlite/src/stream.rs +++ b/crates/storage-sqlite/src/stream.rs @@ -4,16 +4,14 @@ //! `StreamEngine` trait implementation for `SqliteEngine`. //! //! Shards, records, and the sequence counter all live in the one SQLite file, -//! so `init_stream_shards` runs in the caller's catalog transaction. -//! Monotonic sequence numbers come from the `seq_counters` table (there is no -//! PostgreSQL sequence). Retention cleanup compares an RFC 3339 cutoff computed -//! in Rust. +//! so `init_stream_shards` (`stream_lifecycle.rs`) runs in the caller's +//! catalog transaction. Monotonic sequence numbers come from the +//! `seq_counters` table (there is no PostgreSQL sequence). Retention cleanup +//! compares an RFC 3339 cutoff computed in Rust. -use extenddb_core::types::{ - DescribeStreamInput, SequenceNumberRange, Shard, StreamDescription, StreamRecord, StreamStatus, - StreamSummary, StreamViewType, -}; +use extenddb_core::types::{DescribeStreamInput, StreamDescription, StreamRecord}; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage; use extenddb_storage::util::{parse_stream_arn, shard_in_stream, stream_arn}; use extenddb_storage::{StreamEngine, StreamListResult, StreamRecordsResult}; use futures::future::BoxFuture; @@ -21,53 +19,6 @@ use futures::future::BoxFuture; use crate::sqlite_util::format_timestamp; use crate::store::SqliteEngine; -/// Fixed shards per stream (hash-based partition-key assignment). -const SHARDS_PER_STREAM: u32 = 4; - -impl SqliteEngine { - /// Initialize stream shards and set `stream_label`, within the caller's - /// transaction. Returns the assigned stream label. - pub(crate) async fn init_stream_shards( - tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, - account_id: &str, - table_name: &str, - table_id: &str, - ) -> Result { - // Formatted here, not in SQL, so every backend issues the same label - // shape from one function (`extenddb_storage::util::format_stream_label`). - let label = extenddb_storage::util::new_stream_label(); - sqlx::query("UPDATE tables SET stream_label = ? WHERE account_id = ? AND table_name = ?") - .bind(&label) - .bind(account_id) - .bind(table_name) - .execute(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - for i in 0..SHARDS_PER_STREAM { - // Derived from the table id, not its name. A name is shared by - // every account that picks it, while `stream_shards.shard_id` is - // unique across the database. A name of up to 255 bytes also - // overflowed the 65-character ShardId limit - // the AWS SDKs enforce. The table id is a UUID, giving a - // 61-character id. - let shard_id = format!("shardId-{table_id}-{i:016}"); - let start_seq = format!("{:021}", 0); - sqlx::query( - "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ - VALUES (?, ?, ?)", - ) - .bind(&shard_id) - .bind(table_id) - .bind(&start_seq) - .execute(&mut **tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - Ok(label) - } -} - impl StreamEngine for SqliteEngine { fn write_stream_record( &self, @@ -199,92 +150,28 @@ impl StreamEngine for SqliteEngine { ) -> BoxFuture<'_, Result> { let account_id = account_id.to_owned(); let stream_arn = input.stream_arn.clone(); - let limit = input.limit.unwrap_or(100); + let limit = input.limit; let start = input.exclusive_start_shard_id.clone(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - let row: Option<(String, Option, String, String)> = sqlx::query_as( - "SELECT key_schema, stream_specification, table_status, table_id \ - FROM tables WHERE account_id = ? AND table_name = ? AND stream_label = ?", - ) - .bind(&account_id) - .bind(&table_name) - .bind(&stream_label) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let (ks_json, stream_spec_json, table_status, table_id) = - row.ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn))?; - + let (table, ks_json) = self + .stream_table_row(&account_id, &table_name) + .await? + .ok_or_else(|| extenddb_storage::util::stream_not_found(&stream_arn))?; let key_schema = serde_json::from_str(&ks_json) .map_err(|e| StorageError::Internal(e.to_string()))?; - let stream_view_type = stream_spec_json - .as_deref() - .and_then(|s| serde_json::from_str::(s).ok()) - .and_then(|v| { - v.get("StreamViewType") - .and_then(|sv| serde_json::from_value::(sv.clone()).ok()) - }) - .unwrap_or(StreamViewType::KeysOnly); - - let shard_rows: Vec<(String, Option, String, Option)> = - if let Some(ref s) = start { - sqlx::query_as( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM stream_shards WHERE table_id = ? AND shard_id > ? \ - ORDER BY shard_id LIMIT ?", - ) - .bind(&table_id) - .bind(s) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } else { - sqlx::query_as( - "SELECT shard_id, parent_shard_id, starting_sequence_number, ending_sequence_number \ - FROM stream_shards WHERE table_id = ? ORDER BY shard_id LIMIT ?", - ) - .bind(&table_id) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - .map_err(|e| StorageError::Internal(e.to_string()))?; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit.max(0) as usize; - let last_shard = - (shard_rows.len() > limit_usize).then(|| shard_rows[limit_usize - 1].0.clone()); - let shards: Vec = shard_rows - .into_iter() - .take(limit_usize) - .map(|(id, parent, start, end)| Shard { - shard_id: id, - parent_shard_id: parent, - sequence_number_range: SequenceNumberRange { - starting_sequence_number: start, - ending_sequence_number: end, - }, - }) - .collect(); - - let stream_status = if table_status == "DELETING" { - StreamStatus::Disabling - } else { - StreamStatus::Enabled - }; - - Ok(StreamDescription { + let shards = self + .stream_shard_rows(&account_id, Some(&table.table_id)) + .await?; + stream_lineage::describe_stream( stream_arn, - stream_label, - stream_status, - stream_view_type, - table_name, + &table, key_schema, - shards, - last_evaluated_shard_id: last_shard, - }) + stream_label, + &shards, + limit, + start.as_deref(), + ) }) } @@ -299,77 +186,17 @@ impl StreamEngine for SqliteEngine { let table_name = table_name.map(str::to_owned); let start_arn = exclusive_start_stream_arn.map(str::to_owned); Box::pin(async move { - let rows: Vec<(String, String)> = match (table_name.as_deref(), start_arn.as_deref()) { - (Some(tn), Some(arn)) => { - let (_, start_label) = parse_stream_arn(arn)?; - sqlx::query_as( - "SELECT table_name, stream_label FROM tables \ - WHERE account_id = ? AND stream_label IS NOT NULL AND table_name = ? \ - AND stream_label > ? ORDER BY stream_label LIMIT ?", - ) - .bind(&account_id) - .bind(tn) - .bind(&start_label) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - (Some(tn), None) => { - sqlx::query_as( - "SELECT table_name, stream_label FROM tables \ - WHERE account_id = ? AND stream_label IS NOT NULL AND table_name = ? \ - ORDER BY stream_label LIMIT ?", - ) - .bind(&account_id) - .bind(tn) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - (None, Some(arn)) => { - let (start_table, start_label) = parse_stream_arn(arn)?; - sqlx::query_as( - "SELECT table_name, stream_label FROM tables \ - WHERE account_id = ? AND stream_label IS NOT NULL \ - AND (table_name, stream_label) > (?, ?) \ - ORDER BY table_name, stream_label LIMIT ?", - ) - .bind(&account_id) - .bind(&start_table) - .bind(&start_label) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - (None, None) => { - sqlx::query_as( - "SELECT table_name, stream_label FROM tables \ - WHERE account_id = ? AND stream_label IS NOT NULL \ - ORDER BY table_name, stream_label LIMIT ?", - ) - .bind(&account_id) - .bind(limit + 1) - .fetch_all(&self.pool) - .await - } - } - .map_err(|e| StorageError::Internal(e.to_string()))?; - - #[allow(clippy::cast_sign_loss)] - let limit_usize = limit.max(0) as usize; - let summaries: Vec = rows - .iter() - .take(limit_usize) - .map(|(tn, label)| StreamSummary { - stream_arn: stream_arn(&self.region, &account_id, tn, label), - stream_label: label.clone(), - table_name: tn.clone(), - }) - .collect(); - let last = (rows.len() > limit_usize) - .then(|| summaries.last().map(|s| s.stream_arn.clone())) - .flatten(); - Ok((summaries, last)) + let start = start_arn.as_deref().map(parse_stream_arn).transpose()?; + let tables = self.stream_table_rows(&account_id).await?; + let shards = self.stream_shard_rows(&account_id, None).await?; + stream_lineage::list_streams( + &tables, + &shards, + table_name.as_deref(), + limit, + start, + |t, l| stream_arn(&self.region, &account_id, t, l), + ) }) } @@ -411,8 +238,11 @@ impl StreamEngine for SqliteEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))?; + // Only the open shards: they belong to the table's enabled stream. + // A disabled stream's closed shards take no new records. let shards: Vec<(String,)> = sqlx::query_as( - "SELECT shard_id FROM stream_shards WHERE table_id = ? ORDER BY shard_id", + "SELECT shard_id FROM stream_shards \ + WHERE table_id = ? AND ending_sequence_number IS NULL ORDER BY shard_id", ) .bind(&table_id) .fetch_all(&self.pool) @@ -463,31 +293,20 @@ impl StreamEngine for SqliteEngine { let shard_id = shard_id.to_owned(); Box::pin(async move { let (table_name, stream_label) = parse_stream_arn(&stream_arn)?; - let table_id: Option = sqlx::query_scalar( - "SELECT table_id FROM tables \ - WHERE account_id = ? AND table_name = ? AND stream_label = ?", - ) - .bind(&account_id) - .bind(&table_name) - .bind(&stream_label) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - - let Some(table_id) = table_id else { - return Err(extenddb_storage::util::stream_does_not_exist()); + let table = self.stream_table_row(&account_id, &table_name).await?; + let shards = match &table { + Some((t, _)) => { + self.stream_shard_rows(&account_id, Some(&t.table_id)) + .await? + } + None => Vec::new(), }; - let exists: Option<(i64,)> = - sqlx::query_as("SELECT 1 FROM stream_shards WHERE shard_id = ? AND table_id = ?") - .bind(&shard_id) - .bind(&table_id) - .fetch_optional(&self.pool) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if exists.is_none() { - return Err(extenddb_storage::util::shard_not_in_stream(&stream_arn)); - } - Ok(()) + stream_lineage::validate_shard( + table.as_ref().map(|(t, _)| t), + &stream_label, + &shards, + &shard_id, + ) }) } diff --git a/crates/storage-sqlite/src/stream_lifecycle.rs b/crates/storage-sqlite/src/stream_lifecycle.rs new file mode 100644 index 000000000..d67f4045c --- /dev/null +++ b/crates/storage-sqlite/src/stream_lifecycle.rs @@ -0,0 +1,243 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Stream lifecycle on SQLite: creating a table's stream, closing it on +//! `UpdateTable StreamEnabled=false`, and reading the table and shard rows the +//! shared rules in `extenddb_storage::stream_lineage` group into streams. +//! +//! Shards and records live in the catalog file, so every step runs in the +//! caller's transaction under the engine write lock, and a data write cannot +//! interleave with a close. + +use extenddb_core::types::{StreamSpecification, StreamViewType}; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, next_stream_label, shard_id, + stored_specification, stream_change, +}; + +use crate::store::SqliteEngine; + +type ShardTuple = (String, String, Option, String, Option); +/// `(table_id, table_status, stream_label, stream_specification, key_schema)`. +type TableTuple = (String, String, Option, Option, String); +/// `(table_name, table_id, table_status, stream_label, stream_specification)`. +type TableListTuple = (String, String, String, Option, Option); + +fn shard_row((table_id, shard_id, parent, start, end): ShardTuple) -> ShardRow { + ShardRow { + table_id, + shard_id, + parent_shard_id: parent, + starting_sequence_number: start, + ending_sequence_number: end, + } +} + +fn internal(e: impl std::fmt::Display) -> StorageError { + StorageError::Internal(e.to_string()) +} + +impl SqliteEngine { + /// Create a new stream on the table within the caller's transaction: four + /// open shards named for a new label, and `tables.stream_label` set to that + /// label. Returns the label. + pub(crate) async fn init_stream_shards( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + account_id: &str, + table_name: &str, + table_id: &str, + view_type: StreamViewType, + ) -> Result { + let previous: Option = sqlx::query_scalar( + "SELECT stream_label FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let label = next_stream_label(previous.as_deref()); + sqlx::query("UPDATE tables SET stream_label = ? WHERE account_id = ? AND table_name = ?") + .bind(&label) + .bind(account_id) + .bind(table_name) + .execute(&mut **tx) + .await + .map_err(internal)?; + for i in 0..SHARDS_PER_STREAM { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES (?, ?, ?)", + ) + .bind(shard_id(table_id, &label, view_type, i)?) + .bind(table_id) + .bind(format!("{:021}", 0)) + .execute(&mut **tx) + .await + .map_err(internal)?; + } + Ok(label) + } + + /// Close every open shard of the table: its `EndingSequenceNumber` becomes + /// the sequence number of its last record, or its starting number when it + /// holds none (both measured 2026-10-24). + async fn close_stream_shards( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + table_id: &str, + ) -> Result<(), StorageError> { + sqlx::query( + "UPDATE stream_shards SET ending_sequence_number = COALESCE(\ + (SELECT MAX(r.sequence_number) FROM stream_records r \ + WHERE r.shard_id = stream_shards.shard_id), \ + starting_sequence_number) \ + WHERE table_id = ? AND ending_sequence_number IS NULL", + ) + .bind(table_id) + .execute(&mut **tx) + .await + .map_err(internal)?; + Ok(()) + } + + /// Apply an `UpdateTable` `StreamSpecification` within the caller's + /// transaction. + /// + /// Disabling clears the stored specification, so DescribeTable omits + /// `StreamSpecification`, keeps `stream_label` (`LatestStreamArn` keeps + /// naming the disabled stream) and closes the stream's shards. Enabling + /// closes any shard still open (a stream disabled before shards were + /// closed on disable) and creates a new stream. + pub(crate) async fn apply_stream_specification( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + account_id: &str, + table_name: &str, + table_id: &str, + requested: &StreamSpecification, + ) -> Result<(), StorageError> { + let current: Option = sqlx::query_scalar( + "SELECT stream_specification FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&mut **tx) + .await + .map_err(internal)? + .flatten(); + let current: Option = current + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?; + let change = stream_change(table_name, current.as_ref(), requested)?; + let stored = + serde_json::to_string(&stored_specification(change, current.as_ref(), requested)) + .map_err(internal)?; + sqlx::query( + "UPDATE tables SET stream_specification = ? WHERE account_id = ? AND table_name = ?", + ) + .bind(stored) + .bind(account_id) + .bind(table_name) + .execute(&mut **tx) + .await + .map_err(internal)?; + Self::close_stream_shards(tx, table_id).await?; + if let StreamChange::Enable(view_type) = change { + Self::init_stream_shards(tx, account_id, table_name, table_id, view_type).await?; + } + Ok(()) + } + + /// The table's stream columns and its key schema JSON, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let row: Option = sqlx::query_as( + "SELECT table_id, table_status, stream_label, stream_specification, key_schema \ + FROM tables WHERE account_id = ? AND table_name = ?", + ) + .bind(account_id) + .bind(table_name) + .fetch_optional(&self.pool) + .await + .map_err(internal)?; + row.map(|(table_id, table_status, stream_label, spec, key_schema)| { + Ok(( + TableStreamRow { + table_name: table_name.to_owned(), + table_id, + table_status, + stream_label, + stream_specification: spec + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?, + }, + key_schema, + )) + }) + .transpose() + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + let rows: Vec = sqlx::query_as( + "SELECT table_name, table_id, table_status, stream_label, stream_specification \ + FROM tables WHERE account_id = ? AND stream_label IS NOT NULL", + ) + .bind(account_id) + .fetch_all(&self.pool) + .await + .map_err(internal)?; + rows.into_iter() + .map(|(table_name, table_id, table_status, stream_label, spec)| { + Ok(TableStreamRow { + table_name, + table_id, + table_status, + stream_label, + stream_specification: spec + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?, + }) + }) + .collect() + } + + /// The shard rows of one table, or of every table of the account when + /// `table_id` is `None`. + pub(crate) async fn stream_shard_rows( + &self, + account_id: &str, + table_id: Option<&str>, + ) -> Result, StorageError> { + let rows: Vec = sqlx::query_as( + "SELECT s.table_id, s.shard_id, s.parent_shard_id, s.starting_sequence_number, \ + s.ending_sequence_number \ + FROM stream_shards s JOIN tables t ON t.table_id = s.table_id \ + WHERE t.account_id = ? AND (? IS NULL OR s.table_id = ?)", + ) + .bind(account_id) + .bind(table_id) + .bind(table_id) + .fetch_all(&self.pool) + .await + .map_err(internal)?; + Ok(rows.into_iter().map(shard_row).collect()) + } +} + +#[cfg(test)] +#[path = "stream_lifecycle_tests.rs"] +mod tests; diff --git a/crates/storage-sqlite/src/stream_lifecycle_tests.rs b/crates/storage-sqlite/src/stream_lifecycle_tests.rs new file mode 100644 index 000000000..c470b96fb --- /dev/null +++ b/crates/storage-sqlite/src/stream_lifecycle_tests.rs @@ -0,0 +1,357 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Disabling and re-enabling a table's stream on SQLite, against the rows +//! measured on the service on 2026-10-24. + +use extenddb_core::expression::ExpressionMaps; +use extenddb_core::types::{ + DescribeStreamInput, DescribeTableInput, StreamDescription, StreamSpecification, StreamStatus, + StreamViewType, TableDescription, +}; +use extenddb_storage::error::StorageError; +use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, TableEngine}; + +use crate::store::SqliteEngine; + +const ACCOUNT: &str = "123456789012"; + +async fn engine() -> SqliteEngine { + let engine = SqliteEngine::new(":memory:", 1, "us-east-1", 400 * 1024) + .await + .unwrap(); + crate::schema::apply(&engine.pool).await.unwrap(); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES (?, 'stream-test')") + .bind(ACCOUNT) + .execute(&engine.pool) + .await + .unwrap(); + engine +} + +async fn create(engine: &SqliteEngine, name: &str, stream: bool) { + let mut input = serde_json::json!({ + "TableName": name, + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "BillingMode": "PAY_PER_REQUEST", + }); + if stream { + input["StreamSpecification"] = + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_AND_OLD_IMAGES"}); + } + engine + .create_table(ACCOUNT, serde_json::from_value(input).unwrap()) + .await + .unwrap(); + activate(engine, name).await; +} + +/// Finish the control-plane transition the worker would run. +async fn activate(engine: &SqliteEngine, name: &str) { + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_name = ?") + .bind(name) + .execute(&engine.pool) + .await + .unwrap(); +} + +async fn set_stream( + engine: &SqliteEngine, + name: &str, + spec: serde_json::Value, +) -> Result { + let input = serde_json::json!({"TableName": name, "StreamSpecification": spec}); + let out = engine + .update_table(ACCOUNT, serde_json::from_value(input).unwrap()) + .await; + activate(engine, name).await; + out +} + +async fn describe_table(engine: &SqliteEngine, name: &str) -> TableDescription { + engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .unwrap() +} + +async fn describe_stream(engine: &SqliteEngine, arn: &str) -> StreamDescription { + engine + .describe_stream( + ACCOUNT, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .unwrap() +} + +/// Put an item through the data path with the stream capture the engine +/// builds from `view_type`, as a caller holding stale key info would. +async fn put(engine: &SqliteEngine, name: &str, pk: &str, view_type: StreamViewType) { + let key_info = engine.table_key_info(ACCOUNT, name).await.unwrap(); + let item = serde_json::from_value(serde_json::json!({"pk": {"S": pk}})).unwrap(); + let capture = StreamCapture { + view_type, + user_identity: None, + region: "us-east-1".into(), + }; + engine + .put_item( + &key_info, + item, + false, + None, + &ExpressionMaps::default(), + Some(&capture), + ) + .await + .unwrap(); +} + +async fn records_in(engine: &SqliteEngine, shard: &str) -> Vec { + sqlx::query_scalar( + "SELECT sequence_number FROM stream_records WHERE shard_id = ? ORDER BY sequence_number", + ) + .bind(shard) + .fetch_all(&engine.pool) + .await + .unwrap() +} + +#[tokio::test] +async fn disable_closes_every_shard_and_re_enable_creates_a_new_stream() { + let engine = engine().await; + create(&engine, "t482", true).await; + for pk in ["k0", "k1", "k2", "k3", "k4", "k5"] { + put(&engine, "t482", pk, StreamViewType::NewAndOldImages).await; + } + let enabled = describe_table(&engine, "t482").await; + let old_arn = enabled.latest_stream_arn.clone().unwrap(); + let before = describe_stream(&engine, &old_arn).await; + assert!(matches!(before.stream_status, StreamStatus::Enabled)); + assert!( + before + .shards + .iter() + .all(|s| s.sequence_number_range.ending_sequence_number.is_none()) + ); + + let disabled = set_stream(&engine, "t482", serde_json::json!({"StreamEnabled": false})) + .await + .unwrap(); + // The backend keeps `{StreamEnabled: false}` with the closed stream's + // view type; the engine omits it from every response. + assert_eq!( + disabled.stream_specification, + Some(StreamSpecification { + stream_enabled: false, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }) + ); + let mut rendered = disabled.clone(); + rendered.prepare_for_response(); + assert!(rendered.stream_specification.is_none()); + assert_eq!( + disabled.latest_stream_arn.as_deref(), + Some(old_arn.as_str()) + ); + let mut table = describe_table(&engine, "t482").await; + table.prepare_for_response(); + assert!(table.stream_specification.is_none()); + assert_eq!(table.latest_stream_label, enabled.latest_stream_label); + + let closed = describe_stream(&engine, &old_arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::NewAndOldImages); + assert_eq!(closed.shards.len(), 4); + for shard in &closed.shards { + let range = &shard.sequence_number_range; + let last = records_in(&engine, &shard.shard_id).await.pop(); + let expected = last.unwrap_or_else(|| range.starting_sequence_number.clone()); + assert_eq!( + range.ending_sequence_number.as_deref(), + Some(expected.as_str()) + ); + } + + // A write still carrying the enabled view type records nothing. + put(&engine, "t482", "stale", StreamViewType::NewAndOldImages).await; + let total: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stream_records") + .fetch_one(&engine.pool) + .await + .unwrap(); + assert_eq!(total, 6); + + let reenabled = set_stream( + &engine, + "t482", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new_arn = reenabled.latest_stream_arn.clone().unwrap(); + assert_ne!(new_arn, old_arn); + assert!(reenabled.latest_stream_label > enabled.latest_stream_label); + let new = describe_stream(&engine, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.stream_view_type, StreamViewType::KeysOnly); + assert!( + new.shards + .iter() + .all(|s| !closed.shards.iter().any(|o| o.shard_id == s.shard_id)) + ); + + put(&engine, "t482", "after", StreamViewType::KeysOnly).await; + let old_again = describe_stream(&engine, &old_arn).await; + assert!(matches!(old_again.stream_status, StreamStatus::Disabled)); + assert_eq!(old_again.stream_view_type, StreamViewType::NewAndOldImages); + for (a, b) in old_again.shards.iter().zip(&closed.shards) { + assert_eq!(a.shard_id, b.shard_id); + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number + ); + assert_eq!(records_in(&engine, &a.shard_id).await.len(), { + records_in(&engine, &b.shard_id).await.len() + }); + } + let mut on_new = 0; + for s in &new.shards { + on_new += records_in(&engine, &s.shard_id).await.len(); + } + assert_eq!(on_new, 1); + + let (listed, _) = engine + .list_streams(ACCOUNT, Some("t482"), 100, None) + .await + .unwrap(); + let arns: Vec<&str> = listed.iter().map(|s| s.stream_arn.as_str()).collect(); + assert_eq!(arns, [new_arn.as_str(), old_arn.as_str()]); + + // Each ARN accepts only its own shards. + engine + .validate_shard(ACCOUNT, &old_arn, &closed.shards[0].shard_id) + .await + .unwrap(); + for (arn, shard) in [ + (&old_arn, &new.shards[0].shard_id), + (&new_arn, &closed.shards[0].shard_id), + ] { + match engine.validate_shard(ACCOUNT, arn, shard).await { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Shard does not exist"); + } + other => panic!("{other:?}"), + } + } + engine.pool.close().await; +} + +#[tokio::test] +async fn enable_while_enabled_and_disable_without_a_stream_are_refused() { + let engine = engine().await; + create(&engine, "t482", true).await; + create(&engine, "nostream", false).await; + for view in ["NEW_AND_OLD_IMAGES", "KEYS_ONLY"] { + let spec = serde_json::json!({"StreamEnabled": true, "StreamViewType": view}); + match set_stream(&engine, "t482", spec).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table already has an enabled stream: TableName: t482"); + } + other => panic!("{other:?}"), + } + } + match set_stream( + &engine, + "nostream", + serde_json::json!({"StreamEnabled": false}), + ) + .await + { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: nostream"); + } + other => panic!("{other:?}"), + } + set_stream(&engine, "t482", serde_json::json!({"StreamEnabled": false})) + .await + .unwrap(); + match set_stream(&engine, "t482", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: t482"); + } + other => panic!("{other:?}"), + } + // A refused request changes nothing: one stream, four shards. + let shards: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM stream_shards") + .fetch_one(&engine.pool) + .await + .unwrap(); + assert_eq!(shards, 4); + engine.pool.close().await; +} + +#[tokio::test] +async fn a_stream_from_before_labelled_shard_ids_closes_and_is_replaced() { + let engine = engine().await; + create(&engine, "legacy", true).await; + // Rewrite the shards in the shape earlier builds created. + sqlx::query( + "UPDATE stream_shards SET shard_id = 'shardId-' || table_id || '-' || \ + printf('%016d', CAST(substr(shard_id, -2) AS INTEGER))", + ) + .execute(&engine.pool) + .await + .unwrap(); + let arn = describe_table(&engine, "legacy") + .await + .latest_stream_arn + .unwrap(); + put(&engine, "legacy", "k", StreamViewType::NewAndOldImages).await; + let open = describe_stream(&engine, &arn).await; + assert!(matches!(open.stream_status, StreamStatus::Enabled)); + assert_eq!(open.stream_view_type, StreamViewType::NewAndOldImages); + assert!(open.shards.iter().all(|s| { + s.shard_id + .rsplit('-') + .next() + .is_some_and(|suffix| suffix.starts_with("0000000000000")) + })); + + set_stream( + &engine, + "legacy", + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let closed = describe_stream(&engine, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert!( + closed + .shards + .iter() + .all(|s| s.sequence_number_range.ending_sequence_number.is_some()) + ); + let re = set_stream( + &engine, + "legacy", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}), + ) + .await + .unwrap(); + let new = describe_stream(&engine, &re.latest_stream_arn.unwrap()).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.shards.len(), 4); + engine.pool.close().await; +} diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 8adff6173..4677eef49 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -312,49 +312,14 @@ impl SqliteEngine { .await?; } if let Some(spec) = &input.stream_specification { - let j = - serde_json::to_string(spec).map_err(|e| StorageError::Internal(e.to_string()))?; - update_col( + Self::apply_stream_specification( &mut tx, account_id, &input.table_name, - "stream_specification", - &j, + &table_id, + spec, ) .await?; - if spec.stream_enabled { - let existing: Option<(String,)> = - sqlx::query_as("SELECT shard_id FROM stream_shards WHERE table_id = ? LIMIT 1") - .bind(&table_id) - .fetch_optional(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if existing.is_none() { - Self::init_stream_shards(&mut tx, account_id, &input.table_name, &table_id) - .await?; - } else { - let label: Option = sqlx::query_scalar( - "SELECT stream_label FROM tables WHERE account_id = ? AND table_name = ?", - ) - .bind(account_id) - .bind(&input.table_name) - .fetch_one(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - if label.is_none() { - sqlx::query( - "UPDATE tables SET stream_label = ? \ - WHERE account_id = ? AND table_name = ?", - ) - .bind(extenddb_storage::util::new_stream_label()) - .bind(account_id) - .bind(&input.table_name) - .execute(&mut *tx) - .await - .map_err(|e| StorageError::Internal(e.to_string()))?; - } - } - } } // GSI create/delete. diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index b7aec12ff..fe14438b1 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -26,6 +26,7 @@ pub mod snapshot; #[doc(hidden)] #[cfg(feature = "test-hooks")] pub mod snapshot_conformance; +pub mod stream_lineage; #[doc(hidden)] #[cfg(feature = "test-hooks")] pub mod test_pause; diff --git a/crates/storage/src/stream_lineage.rs b/crates/storage/src/stream_lineage.rs new file mode 100644 index 000000000..3fb18e8e7 --- /dev/null +++ b/crates/storage/src/stream_lineage.rs @@ -0,0 +1,578 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The streams a table has had, and which of them a shard belongs to. +//! +//! Disabling a table's stream closes it: the stream reports `DISABLED`, every +//! shard gets an `EndingSequenceNumber`, and the stream stays describable and +//! listed. Enabling it again creates a new stream with a new label and new +//! shards. A table therefore has a sequence of streams, and only the newest +//! one can be enabled. +//! +//! Each stream's identity is carried in its shard ids rather than in a catalog +//! row, so no backend needs a schema change: a shard id is +//! `shardId-{table_id}-{label_millis:013}{view_type:1}{index:02}`, where +//! `label_millis` is the stream label as epoch milliseconds and `view_type` +//! one digit for the stream's `StreamViewType`. A shard id written before +//! this scheme (`shardId-{table_id}-{index}`, all zeros but the index) +//! belongs to the table's current stream, as it always did. +//! +//! Every backend reads its table row and shard rows and hands them to the +//! functions here, so the grouping, the status rule, the paging order and the +//! refusal messages are written once. + +use extenddb_core::types::{ + KeySchemaElement, SequenceNumberRange, Shard, StreamDescription, StreamSpecification, + StreamStatus, StreamSummary, StreamViewType, +}; + +use crate::error::StorageError; +use crate::util::{ + format_stream_label, shard_does_not_exist, stream_does_not_exist, stream_not_found, +}; + +/// Fixed shards per stream (hash-based partition-key assignment). +pub const SHARDS_PER_STREAM: u32 = 4; + +/// A `CreateTable` request with a disabled `StreamSpecification` dropped. +/// +/// The service does not store `{StreamEnabled: false}`: `DescribeTable` on +/// such a table has no `StreamSpecification`, `LatestStreamArn` or +/// `LatestStreamLabel`, and the table answers `UpdateTable` as one created +/// without the member (measured 2026-10-24). Every backend passes its request +/// through here before storing it. +#[must_use] +pub fn without_disabled_specification( + mut input: extenddb_core::types::CreateTableInput, +) -> extenddb_core::types::CreateTableInput { + if input + .stream_specification + .as_ref() + .is_some_and(|s| !s.stream_enabled) + { + input.stream_specification = None; + } + input +} + +/// What an `UpdateTable` `StreamSpecification` asks of the table's stream. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamChange { + /// Create a new stream with this view type. + Enable(StreamViewType), + /// Close the enabled stream. + Disable, +} + +/// Decide a `StreamSpecification` change against the table's stored one. +/// +/// # Errors +/// +/// `Validation` with the service's sentence when the request enables a stream +/// on a table whose stream is enabled (whatever the view type), or disables +/// one on a table with no enabled stream (never had one, or already +/// disabled). Both measured 2026-10-24. +pub fn stream_change( + table_name: &str, + current: Option<&StreamSpecification>, + requested: &StreamSpecification, +) -> Result { + let enabled = current.is_some_and(|s| s.stream_enabled); + match (requested.stream_enabled, enabled) { + (true, true) => Err(StorageError::Validation(format!( + "Table already has an enabled stream: TableName: {table_name}" + ))), + (false, false) => Err(StorageError::Validation(format!( + "Table has no stream to disable: TableName: {table_name}" + ))), + // Unreachable through the engine, which refuses an enable without a + // view type before storage with this same sentence. The service's + // sentence for that case, measured 2026-10-24, is "One or more + // parameter values were invalid: If stream is being enabled then + // UpdateViewType is required", and it precedes the already-enabled + // check; the engine's sentence pre-dates this change and is not + // changed here. + (true, false) => requested + .stream_view_type + .map(StreamChange::Enable) + .ok_or_else(|| { + StorageError::Validation( + "StreamViewType must be specified when StreamEnabled is true".to_owned(), + ) + }), + (false, true) => Ok(StreamChange::Disable), + } +} + +/// The specification to store once a stream change is decided: the request's +/// own on an enable, and on a disable `{StreamEnabled: false}` carrying the +/// closed stream's view type, so a legacy stream (whose view type lives only +/// here) still reports it once disabled. Responses omit a disabled +/// specification (`TableDescription::prepare_for_response`). +#[must_use] +pub fn stored_specification( + change: StreamChange, + current: Option<&StreamSpecification>, + requested: &StreamSpecification, +) -> StreamSpecification { + match change { + StreamChange::Enable(_) => requested.clone(), + StreamChange::Disable => StreamSpecification { + stream_enabled: false, + stream_view_type: current.and_then(|c| c.stream_view_type), + }, + } +} + +/// The label for a new stream: now, or one millisecond after the table's +/// previous label when the clock has not moved past it, so two streams of one +/// table never share an ARN. +#[must_use] +pub fn next_stream_label(previous: Option<&str>) -> String { + let now = crate::util::new_stream_label(); + match previous { + Some(prev) if now.as_str() <= prev => label_millis(prev) + .and_then(|m| millis_label(m + 1)) + .unwrap_or(now), + _ => now, + } +} + +/// The epoch milliseconds a stream label names, or `None` for a string that is +/// not in the label shape. +#[must_use] +pub fn label_millis(label: &str) -> Option { + let b = label.as_bytes(); + let seps = [ + (4, b'-'), + (7, b'-'), + (10, b'T'), + (13, b':'), + (16, b':'), + (19, b'.'), + ]; + if b.len() != 23 || seps.iter().any(|&(i, c)| b[i] != c) { + return None; + } + let num = |from: usize, to: usize| -> Option { + let s = label.get(from..to)?; + s.bytes() + .all(|c| c.is_ascii_digit()) + .then(|| s.parse().ok())? + }; + let month = time::Month::try_from(u8::try_from(num(5, 7)?).ok()?).ok()?; + let date = time::Date::from_calendar_date( + i32::from(num(0, 4)?), + month, + u8::try_from(num(8, 10)?).ok()?, + ) + .ok()?; + let at = time::Time::from_hms_milli( + u8::try_from(num(11, 13)?).ok()?, + u8::try_from(num(14, 16)?).ok()?, + u8::try_from(num(17, 19)?).ok()?, + num(20, 23)?, + ) + .ok()?; + i64::try_from(date.with_time(at).assume_utc().unix_timestamp_nanos() / 1_000_000).ok() +} + +fn millis_label(millis: i64) -> Option { + time::OffsetDateTime::from_unix_timestamp_nanos(i128::from(millis) * 1_000_000) + .ok() + .map(format_stream_label) +} + +fn view_digit(view_type: StreamViewType) -> char { + match view_type { + StreamViewType::KeysOnly => '1', + StreamViewType::NewImage => '2', + StreamViewType::OldImage => '3', + StreamViewType::NewAndOldImages => '4', + } +} + +fn digit_view(digit: u8) -> Option { + match digit { + b'1' => Some(StreamViewType::KeysOnly), + b'2' => Some(StreamViewType::NewImage), + b'3' => Some(StreamViewType::OldImage), + b'4' => Some(StreamViewType::NewAndOldImages), + _ => None, + } +} + +/// The id of shard `index` of the stream `label` on table `table_id`. +/// +/// Derived from the table id, not its name: a name is shared by every account +/// that picks it and reused by delete-and-recreate, while a shard id must be +/// unique across a backend's shard store. 8 + 36 + 1 + 16 = 61 characters for +/// a UUID table id, inside the 65-character limit the AWS SDKs enforce on +/// `ShardId`. +/// +/// # Errors +/// +/// `Internal` when `label` is not in the label shape; every label comes from +/// [`next_stream_label`] or an earlier stored label. +pub fn shard_id( + table_id: &str, + label: &str, + view_type: StreamViewType, + index: u32, +) -> Result { + let millis = label_millis(label) + .ok_or_else(|| StorageError::Internal(format!("malformed stream label {label}")))?; + Ok(format!( + "shardId-{table_id}-{millis:013}{}{index:02}", + view_digit(view_type) + )) +} + +/// Which stream of its table a shard id belongs to. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ShardStream { + /// An id from before streams were named in shard ids: the table's current + /// stream. + Legacy, + /// An id naming its stream's label and view type. + Labelled { + label: String, + view_type: StreamViewType, + }, +} + +/// Parse a shard id of table `table_id`: its stream and its index within the +/// stream. `None` when the id is not one of this table's. +#[must_use] +pub fn parse_shard_id(table_id: &str, shard_id: &str) -> Option<(ShardStream, u32)> { + let suffix = shard_id + .strip_prefix("shardId-")? + .strip_prefix(table_id)? + .strip_prefix('-')?; + if suffix.is_empty() || !suffix.bytes().all(|c| c.is_ascii_digit()) { + return None; + } + if suffix.len() == 16 && !suffix.starts_with("0000000000000") { + let millis: i64 = suffix[..13].parse().ok()?; + let view_type = digit_view(suffix.as_bytes()[13])?; + let index: u32 = suffix[14..].parse().ok()?; + return Some(( + ShardStream::Labelled { + label: millis_label(millis)?, + view_type, + }, + index, + )); + } + Some((ShardStream::Legacy, suffix.parse().ok()?)) +} + +/// One shard row as a backend stores it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShardRow { + pub table_id: String, + pub shard_id: String, + pub parent_shard_id: Option, + pub starting_sequence_number: String, + pub ending_sequence_number: Option, +} + +/// The stream-relevant columns of one table row. +#[derive(Debug, Clone)] +pub struct TableStreamRow { + pub table_name: String, + pub table_id: String, + pub table_status: String, + /// `LatestStreamLabel`: the newest stream, enabled or not. + pub stream_label: Option, + pub stream_specification: Option, +} + +/// One stream of a table and its shards, sorted by shard id. +#[derive(Debug, Clone)] +pub struct TableStream { + pub label: String, + pub view_type: StreamViewType, + pub shards: Vec, +} + +/// Group a table's shards into its streams, oldest first. +/// +/// Legacy shard ids (any id of this table that is not in the labelled shape) +/// take the table's current label and the view type the stored specification +/// carries, enabled or disabled (`KEYS_ONLY` when it carries none, which is +/// what DescribeStream reported for them before), unless a labelled stream +/// already holds that label. +#[must_use] +pub fn table_streams(table: &TableStreamRow, shards: &[ShardRow]) -> Vec { + let mut streams: Vec = Vec::new(); + let mut legacy: Vec = Vec::new(); + for row in shards.iter().filter(|r| r.table_id == table.table_id) { + match parse_shard_id(&table.table_id, &row.shard_id) { + Some((ShardStream::Labelled { label, view_type }, _)) => { + match streams.iter_mut().find(|s| s.label == label) { + Some(s) => s.shards.push(row.clone()), + None => streams.push(TableStream { + label, + view_type, + shards: vec![row.clone()], + }), + } + } + // Ids from before streams were named in shard ids: the + // `shardId-{table_id}-{index}` form, and the + // `shardId-{table_name}-{index}` form SQLite and PostgreSQL wrote + // before v0.1.14. The rows are already this table's (filtered by + // table id above), so any id that does not parse is a legacy one. + Some((ShardStream::Legacy, _)) | None => legacy.push(row.clone()), + } + } + if let Some(current) = table.stream_label.as_deref() + && !legacy.is_empty() + && !streams.iter().any(|s| s.label == current) + { + let view_type = table + .stream_specification + .as_ref() + .and_then(|s| s.stream_view_type) + .unwrap_or(StreamViewType::KeysOnly); + streams.push(TableStream { + label: current.to_owned(), + view_type, + shards: legacy, + }); + } + for s in &mut streams { + s.shards.sort_by(|a, b| a.shard_id.cmp(&b.shard_id)); + } + streams.sort_by(|a, b| a.label.cmp(&b.label)); + streams +} + +/// The ids of the open shards of the stream `label` of table `table_id`: the +/// labelled rows carrying that label, or the legacy rows when no labelled row +/// carries it (a legacy stream's label is the table's label at the time it was +/// read). A disable closes exactly these, so a disable that applies after an +/// overlapping enable has committed does not close the stream the enable just +/// opened. +#[must_use] +pub fn open_shard_ids_of(table_id: &str, label: &str, shards: &[ShardRow]) -> Vec { + let rows: Vec<&ShardRow> = shards.iter().filter(|r| r.table_id == table_id).collect(); + let labelled_exists = rows.iter().any(|r| { + matches!( + parse_shard_id(table_id, &r.shard_id), + Some((ShardStream::Labelled { label: l, .. }, _)) if l == label + ) + }); + let mut ids: Vec = rows + .into_iter() + .filter(|r| r.ending_sequence_number.is_none()) + .filter(|r| match parse_shard_id(table_id, &r.shard_id) { + Some((ShardStream::Labelled { label: l, .. }, _)) => l == label, + Some((ShardStream::Legacy, _)) | None => !labelled_exists, + }) + .map(|r| r.shard_id.clone()) + .collect(); + ids.sort(); + ids +} + +/// A stream's status: only the table's newest stream, while its specification +/// is enabled, is `ENABLED` (`DISABLING` once the table is being deleted); +/// every other stream of the table is `DISABLED`. +#[must_use] +pub fn stream_status(table: &TableStreamRow, label: &str) -> StreamStatus { + let current = table.stream_label.as_deref() == Some(label) + && table + .stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled); + if !current { + StreamStatus::Disabled + } else if table.table_status == "DELETING" { + StreamStatus::Disabling + } else { + StreamStatus::Enabled + } +} + +/// Whether the stream `label` an enable just created is still the table's +/// enabled stream, read from the catalog after its shards were inserted. On +/// the backends whose catalog and shards are two commits, a disable that reads +/// the row between the enable's commit and its shard inserts closes the shards +/// it finds at that instant, so the enable re-reads the row afterwards and +/// closes what it inserted when the answer is `false` (the label moved on, the +/// specification is no longer enabled, or the table is gone). +#[must_use] +pub fn enable_still_current(table: Option<&TableStreamRow>, label: &str) -> bool { + table.is_some_and(|t| { + t.stream_label.as_deref() == Some(label) + && t.stream_specification + .as_ref() + .is_some_and(|s| s.stream_enabled) + }) +} + +/// The shard ids of the table's open shards: the ones its enabled stream +/// writes to. Empty when the stream is disabled. +#[must_use] +pub fn open_shard_ids(shards: &[ShardRow]) -> Vec { + let mut ids: Vec = shards + .iter() + .filter(|r| r.ending_sequence_number.is_none()) + .map(|r| r.shard_id.clone()) + .collect(); + ids.sort(); + ids +} + +/// Build a `DescribeStream` answer from the table row and all its shard rows. +/// +/// # Errors +/// +/// `TableNotFound` with the stream-not-found message when the table has no +/// stream with this label. +#[allow(clippy::too_many_arguments)] +pub fn describe_stream( + stream_arn: String, + table: &TableStreamRow, + key_schema: Vec, + label: String, + shards: &[ShardRow], + limit: Option, + exclusive_start_shard_id: Option<&str>, +) -> Result { + let stream = table_streams(table, shards) + .into_iter() + .find(|s| s.label == label) + .ok_or_else(|| stream_not_found(&stream_arn))?; + let limit = usize::try_from(limit.unwrap_or(100)).unwrap_or(0).max(1); + let mut remaining: Vec<&ShardRow> = stream + .shards + .iter() + .filter(|r| exclusive_start_shard_id.is_none_or(|s| r.shard_id.as_str() > s)) + .collect(); + let last_evaluated_shard_id = + (remaining.len() > limit).then(|| remaining[limit - 1].shard_id.clone()); + remaining.truncate(limit); + Ok(StreamDescription { + stream_status: stream_status(table, &label), + stream_view_type: stream.view_type, + stream_arn, + stream_label: label, + table_name: table.table_name.clone(), + key_schema, + shards: remaining + .into_iter() + .map(|r| Shard { + shard_id: r.shard_id.clone(), + parent_shard_id: r.parent_shard_id.clone(), + sequence_number_range: SequenceNumberRange { + starting_sequence_number: r.starting_sequence_number.clone(), + ending_sequence_number: r.ending_sequence_number.clone(), + }, + }) + .collect(), + last_evaluated_shard_id, + }) +} + +/// Check that `shard_id` is a shard of the stream `label` of `table`. +/// +/// # Errors +/// +/// [`crate::util::STREAM_DOES_NOT_EXIST`] when the table has no such stream, +/// and [`crate::util::SHARD_DOES_NOT_EXIST`] when the stream exists but the +/// shard is not one of its own (measured 2026-10-24 for a shard of the table's +/// other stream and for a well-formed shard id the account never had). +pub fn validate_shard( + table: Option<&TableStreamRow>, + label: &str, + shards: &[ShardRow], + shard_id: &str, +) -> Result<(), StorageError> { + let stream = table + .and_then(|t| { + table_streams(t, shards) + .into_iter() + .find(|s| s.label == label) + }) + .ok_or_else(stream_does_not_exist)?; + if stream.shards.iter().any(|r| r.shard_id == shard_id) { + Ok(()) + } else { + Err(shard_does_not_exist()) + } +} + +/// The sentence for an `ExclusiveStartStreamArn` that names a stream of a +/// table other than `TableName` (measured 2026-10-24). +#[must_use] +pub fn invalid_exclusive_start_stream_arn(arn: &str) -> StorageError { + StorageError::Validation(format!( + "One or more parameter values were invalid: ExclusiveStartStreamArn provided is invalid: {arn}" + )) +} + +/// Build a `ListStreams` page from the account's tables and their shards. +/// +/// Every stream of every table is listed, disabled ones included, in the +/// order the service answered on 2026-10-24: table names descending, and a +/// table's streams newest first. `exclusive_start` is the (table name, label) +/// of the previous page's last stream; `arn` formats a stream ARN. +/// `LastEvaluatedStreamArn` is present whenever `limit` cut the page, even +/// when no stream follows: with two streams and `Limit=1`, page two carries it +/// and page three is empty (measured 2026-10-24). +/// +/// # Errors +/// +/// `Validation` when `exclusive_start` names a table other than `table_name`. +pub fn list_streams( + tables: &[TableStreamRow], + shards: &[ShardRow], + table_name: Option<&str>, + limit: i64, + exclusive_start: Option<(String, String)>, + arn: impl Fn(&str, &str) -> String, +) -> Result<(Vec, Option), StorageError> { + if let (Some(name), Some((start_table, start_label))) = (table_name, &exclusive_start) + && start_table != name + { + return Err(invalid_exclusive_start_stream_arn(&arn( + start_table, + start_label, + ))); + } + let mut listed: Vec<(String, String)> = tables + .iter() + .filter(|t| table_name.is_none_or(|n| t.table_name == n)) + .flat_map(|t| { + table_streams(t, shards) + .into_iter() + .map(|s| (t.table_name.clone(), s.label)) + }) + .collect(); + listed.sort_by(|a, b| b.cmp(a)); + if let Some(start) = exclusive_start { + listed.retain(|entry| *entry < start); + } + let limit = usize::try_from(limit).unwrap_or(0).max(1); + let last = (listed.len() >= limit).then(|| { + let (t, l) = &listed[limit - 1]; + arn(t, l) + }); + listed.truncate(limit); + let summaries = listed + .into_iter() + .map(|(table_name, label)| StreamSummary { + stream_arn: arn(&table_name, &label), + stream_label: label, + table_name, + }) + .collect(); + Ok((summaries, last)) +} + +#[cfg(test)] +#[path = "stream_lineage_tests.rs"] +mod tests; diff --git a/crates/storage/src/stream_lineage_tests.rs b/crates/storage/src/stream_lineage_tests.rs new file mode 100644 index 000000000..b4e4daa02 --- /dev/null +++ b/crates/storage/src/stream_lineage_tests.rs @@ -0,0 +1,465 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +use super::*; + +const TID: &str = "0865b556-c8dd-433c-bb2a-f6a8fb6b6e65"; +const OLD: &str = "2026-10-24T08:54:14.037"; +const NEW: &str = "2026-10-24T08:54:32.805"; + +fn spec(enabled: bool, view: Option) -> StreamSpecification { + StreamSpecification { + stream_enabled: enabled, + stream_view_type: view, + } +} + +fn table(label: Option<&str>, spec: Option) -> TableStreamRow { + TableStreamRow { + table_name: "t482".to_owned(), + table_id: TID.to_owned(), + table_status: "ACTIVE".to_owned(), + stream_label: label.map(str::to_owned), + stream_specification: spec, + } +} + +fn row(shard_id: String, end: Option<&str>) -> ShardRow { + ShardRow { + table_id: TID.to_owned(), + shard_id, + parent_shard_id: None, + starting_sequence_number: "0".repeat(21), + ending_sequence_number: end.map(str::to_owned), + } +} + +fn stream_rows(label: &str, view: StreamViewType, end: Option<&str>) -> Vec { + (0..SHARDS_PER_STREAM) + .map(|i| row(shard_id(TID, label, view, i).unwrap(), end)) + .collect() +} + +#[test] +fn the_two_refusals_are_the_measured_sentences() { + let enabled = spec(true, Some(StreamViewType::NewAndOldImages)); + for requested in [ + spec(true, Some(StreamViewType::NewAndOldImages)), + spec(true, Some(StreamViewType::KeysOnly)), + ] { + match stream_change("t482-1792832053", Some(&enabled), &requested) { + Err(StorageError::Validation(m)) => assert_eq!( + m, + "Table already has an enabled stream: TableName: t482-1792832053" + ), + other => panic!("{other:?}"), + } + } + let disabled = spec(false, None); + for current in [None, Some(&disabled)] { + match stream_change("x482-nostream-1792832053", current, &disabled) { + Err(StorageError::Validation(m)) => assert_eq!( + m, + "Table has no stream to disable: TableName: x482-nostream-1792832053" + ), + other => panic!("{other:?}"), + } + } + assert_eq!( + stream_change("t", Some(&enabled), &disabled).unwrap(), + StreamChange::Disable + ); + assert_eq!( + stream_change("t", Some(&disabled), &enabled).unwrap(), + StreamChange::Enable(StreamViewType::NewAndOldImages) + ); + assert_eq!( + stream_change("t", None, &enabled).unwrap(), + StreamChange::Enable(StreamViewType::NewAndOldImages) + ); +} + +#[test] +fn labels_round_trip_through_shard_ids() { + assert_eq!(label_millis("1970-01-01T00:00:00.001"), Some(1)); + assert_eq!(label_millis("2026-10-24T08:54:14.03"), None); + assert_eq!(label_millis("2026-10-24 08:54:14.037"), None); + assert_eq!(label_millis("2026-13-24T08:54:14.037"), None); + for view in [ + StreamViewType::KeysOnly, + StreamViewType::NewImage, + StreamViewType::OldImage, + StreamViewType::NewAndOldImages, + ] { + let id = shard_id(TID, OLD, view, 3).unwrap(); + assert!(id.len() <= 65, "{id}"); + assert_eq!( + parse_shard_id(TID, &id), + Some(( + ShardStream::Labelled { + label: OLD.to_owned(), + view_type: view + }, + 3 + )) + ); + } + // Ids written before streams were named in shard ids: 16 digits on SQLite + // and PostgreSQL, 12 on MongoDB and Cassandra. + for legacy in [ + format!("shardId-{TID}-{:016}", 2), + format!("shardId-{TID}-{:012}", 2), + ] { + assert_eq!(parse_shard_id(TID, &legacy), Some((ShardStream::Legacy, 2))); + } + assert_eq!( + parse_shard_id( + "other", + &shard_id(TID, OLD, StreamViewType::KeysOnly, 0).unwrap() + ), + None + ); + assert!(shard_id(TID, "not a label", StreamViewType::KeysOnly, 0).is_err()); +} + +#[test] +fn a_new_label_is_later_than_the_previous_one() { + let far_future = "2999-01-01T00:00:00.000"; + assert_eq!( + next_stream_label(Some(far_future)), + "2999-01-01T00:00:00.001" + ); + let fresh = next_stream_label(Some(OLD)); + assert!(fresh.as_str() > OLD, "{fresh}"); + assert!(label_millis(&next_stream_label(None)).is_some()); +} + +#[test] +fn a_disabled_stream_stays_describable_with_its_ending_numbers() { + let mut shards = stream_rows(OLD, StreamViewType::NewAndOldImages, Some("1")); + shards[3].ending_sequence_number = Some(shards[3].starting_sequence_number.clone()); + let t = table(Some(OLD), None); + let d = describe_stream( + "arn".into(), + &t, + Vec::new(), + OLD.into(), + &shards, + None, + None, + ) + .unwrap(); + assert!(matches!(d.stream_status, StreamStatus::Disabled)); + assert_eq!(d.stream_view_type, StreamViewType::NewAndOldImages); + assert_eq!(d.shards.len(), 4); + assert!(d.last_evaluated_shard_id.is_none()); + assert!( + d.shards + .iter() + .all(|s| s.sequence_number_range.ending_sequence_number.is_some()) + ); + + // Limit=2 pages through the disabled stream as through an enabled one. + let p1 = describe_stream( + "arn".into(), + &t, + Vec::new(), + OLD.into(), + &shards, + Some(2), + None, + ) + .unwrap(); + assert_eq!(p1.shards.len(), 2); + let start = p1.last_evaluated_shard_id.clone().unwrap(); + assert_eq!(start, p1.shards[1].shard_id); + let p2 = describe_stream( + "arn".into(), + &t, + Vec::new(), + OLD.into(), + &shards, + Some(2), + Some(&start), + ) + .unwrap(); + assert_eq!(p2.shards.len(), 2); + assert!(p2.last_evaluated_shard_id.is_none()); +} + +#[test] +fn a_re_enabled_table_has_two_streams_and_only_the_new_one_is_enabled() { + let mut shards = stream_rows(OLD, StreamViewType::NewAndOldImages, Some("5")); + shards.extend(stream_rows(NEW, StreamViewType::KeysOnly, None)); + let t = table(Some(NEW), Some(spec(true, Some(StreamViewType::KeysOnly)))); + let streams = table_streams(&t, &shards); + assert_eq!(streams.len(), 2); + assert_eq!(streams[0].label, OLD); + assert!(matches!(stream_status(&t, OLD), StreamStatus::Disabled)); + assert!(matches!(stream_status(&t, NEW), StreamStatus::Enabled)); + let old = describe_stream("a".into(), &t, Vec::new(), OLD.into(), &shards, None, None).unwrap(); + assert_eq!(old.stream_view_type, StreamViewType::NewAndOldImages); + let new = describe_stream("a".into(), &t, Vec::new(), NEW.into(), &shards, None, None).unwrap(); + assert!( + new.shards + .iter() + .all(|s| !old.shards.iter().any(|o| o.shard_id == s.shard_id)) + ); + assert_eq!(open_shard_ids(&shards).len(), 4); + + // Each ARN accepts only its own stream's shards. + validate_shard(Some(&t), OLD, &shards, &old.shards[0].shard_id).unwrap(); + for (label, shard) in [ + (OLD, &new.shards[0].shard_id), + (NEW, &old.shards[0].shard_id), + ] { + match validate_shard(Some(&t), label, &shards, shard) { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Shard does not exist"); + } + other => panic!("{other:?}"), + } + } + match validate_shard(Some(&t), "2020-01-01T00:00:00.000", &shards, "s") { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, "Requested resource not found: Stream does not exist"); + } + other => panic!("{other:?}"), + } +} + +/// SQLite and PostgreSQL wrote `shardId-{table_name}-{index:016}` before +/// v0.1.14; those rows are this table's shards and its stream must stay +/// describable after an upgrade. +#[test] +fn name_based_shard_ids_from_before_v0_1_14_are_the_current_stream() { + let rows: Vec = (0..4) + .map(|i| row(format!("shardId-t482-{i:016}"), None)) + .collect(); + let t = table(Some(OLD), Some(spec(true, Some(StreamViewType::NewImage)))); + let streams = table_streams(&t, &rows); + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].label, OLD); + assert_eq!(streams[0].view_type, StreamViewType::NewImage); + assert_eq!(streams[0].shards.len(), 4); + validate_shard(Some(&t), OLD, &rows, "shardId-t482-0000000000000002").unwrap(); + let d = describe_stream("arn".into(), &t, Vec::new(), OLD.into(), &rows, None, None).unwrap(); + assert_eq!(d.shards.len(), 4); + assert!(matches!(d.stream_status, StreamStatus::Enabled)); + // A shard row of another table never reaches the grouping. + let foreign = ShardRow { + table_id: "other".to_owned(), + ..row("shardId-other-0000000000000000".to_owned(), None) + }; + assert_eq!(table_streams(&t, &[foreign]).len(), 0); +} + +#[test] +fn legacy_shards_belong_to_the_current_label_until_a_new_stream_takes_it() { + let legacy: Vec = (0..4) + .map(|i| row(format!("shardId-{TID}-{i:016}"), None)) + .collect(); + let t = table(Some(OLD), Some(spec(true, Some(StreamViewType::NewImage)))); + let streams = table_streams(&t, &legacy); + assert_eq!(streams.len(), 1); + assert_eq!(streams[0].label, OLD); + assert_eq!(streams[0].view_type, StreamViewType::NewImage); + + let mut both = legacy; + both.extend(stream_rows(NEW, StreamViewType::NewImage, None)); + let t = table(Some(NEW), Some(spec(true, Some(StreamViewType::NewImage)))); + let streams = table_streams(&t, &both); + assert_eq!( + streams.len(), + 1, + "the legacy stream's label is no longer known" + ); + assert_eq!(streams[0].label, NEW); + assert!(streams[0].shards.iter().all(|s| s.shard_id.len() == 61)); +} + +#[test] +fn list_streams_orders_tables_descending_and_streams_newest_first() { + let mut shards = stream_rows(OLD, StreamViewType::NewAndOldImages, Some("5")); + shards.extend(stream_rows(NEW, StreamViewType::NewAndOldImages, None)); + let other_id = "11111111-2222-3333-4444-555555555555"; + shards.extend((0..4).map(|i| ShardRow { + table_id: other_id.to_owned(), + shard_id: shard_id(other_id, OLD, StreamViewType::KeysOnly, i).unwrap(), + ..row(String::new(), None) + })); + let tables = vec![ + table( + Some(NEW), + Some(spec(true, Some(StreamViewType::NewAndOldImages))), + ), + TableStreamRow { + table_name: "a-table".to_owned(), + table_id: other_id.to_owned(), + ..table(Some(OLD), Some(spec(true, Some(StreamViewType::KeysOnly)))) + }, + ]; + let arn = |t: &str, l: &str| format!("{t}/{l}"); + let (all, last) = list_streams(&tables, &shards, None, 100, None, arn).unwrap(); + let got: Vec = all.iter().map(|s| s.stream_arn.clone()).collect(); + assert_eq!( + got, + [ + format!("t482/{NEW}"), + format!("t482/{OLD}"), + format!("a-table/{OLD}") + ] + ); + assert!(last.is_none()); + + let (only, last) = list_streams(&tables, &shards, Some("t482"), 100, None, arn).unwrap(); + assert_eq!(only.len(), 2); + assert_eq!(only[0].stream_label, NEW); + assert!(last.is_none()); + + // Limit=1 over the two streams of one table: three pages, the last ARN + // present on every page the limit cut, the third page empty. + let page = |start: Option<&str>| { + list_streams( + &tables, + &shards, + Some("t482"), + 1, + start.map(|l| ("t482".to_owned(), l.to_owned())), + arn, + ) + .unwrap() + }; + let (p1, last) = page(None); + assert_eq!(p1[0].stream_label, NEW); + assert_eq!(last.as_deref(), Some(format!("t482/{NEW}").as_str())); + let (p2, last) = page(Some(NEW)); + assert_eq!(p2[0].stream_label, OLD); + assert_eq!(last.as_deref(), Some(format!("t482/{OLD}").as_str())); + let (p3, last) = page(Some(OLD)); + assert!(p3.is_empty()); + assert!(last.is_none()); + + // No Limit: after the older stream nothing, after the newest the older. + let from = |l: &str| { + list_streams( + &tables, + &shards, + Some("t482"), + 100, + Some(("t482".to_owned(), l.to_owned())), + arn, + ) + .unwrap() + }; + let (after_old, last) = from(OLD); + assert!(after_old.is_empty()); + assert!(last.is_none()); + let (after_new, last) = from(NEW); + assert_eq!(after_new.len(), 1); + assert_eq!(after_new[0].stream_label, OLD); + assert!(last.is_none()); + + // A start ARN of another table, with TableName, is refused. + match list_streams( + &tables, + &shards, + Some("t482"), + 100, + Some(("a-table".to_owned(), OLD.to_owned())), + arn, + ) { + Err(StorageError::Validation(m)) => assert_eq!( + m, + format!( + "One or more parameter values were invalid: ExclusiveStartStreamArn provided is invalid: a-table/{OLD}" + ) + ), + other => panic!("{other:?}"), + } +} + +#[test] +fn a_disabled_create_table_specification_is_not_stored() { + let make = |spec: Option| extenddb_core::types::CreateTableInput { + table_name: "t".to_owned(), + stream_specification: spec, + ..Default::default() + }; + assert_eq!( + without_disabled_specification(make(Some(spec(false, None)))).stream_specification, + None + ); + let enabled = spec(true, Some(StreamViewType::KeysOnly)); + assert_eq!( + without_disabled_specification(make(Some(enabled.clone()))).stream_specification, + Some(enabled) + ); + assert_eq!( + without_disabled_specification(make(None)).stream_specification, + None + ); +} + +/// A disable closes the shards of the stream it decided on and no other, +/// including when an overlapping enable has already opened the next stream. +#[test] +fn a_close_selects_only_the_shards_of_its_label() { + let mut shards = stream_rows(OLD, StreamViewType::NewAndOldImages, None); + shards.extend(stream_rows(NEW, StreamViewType::KeysOnly, None)); + let old_ids = open_shard_ids_of(TID, OLD, &shards); + assert_eq!(old_ids.len(), 4); + assert!( + old_ids + .iter() + .all(|id| shards[..4].iter().any(|r| &r.shard_id == id)) + ); + let new_ids = open_shard_ids_of(TID, NEW, &shards); + assert_eq!(new_ids.len(), 4); + assert!(old_ids.iter().all(|id| !new_ids.contains(id))); + // A label no shard carries selects nothing. + assert!(open_shard_ids_of(TID, "2020-01-01T00:00:00.000", &shards).is_empty()); + // Closed shards are never selected again. + shards[0].ending_sequence_number = Some("5".to_owned()); + assert_eq!(open_shard_ids_of(TID, OLD, &shards).len(), 3); + + // Legacy rows belong to whatever label the change read, until a labelled + // stream carries that label. + let legacy: Vec = (0..4) + .map(|i| row(format!("shardId-t482-{i:016}"), None)) + .collect(); + assert_eq!(open_shard_ids_of(TID, OLD, &legacy).len(), 4); + let mut both = legacy.clone(); + both.extend(stream_rows(OLD, StreamViewType::KeysOnly, None)); + assert_eq!(open_shard_ids_of(TID, OLD, &both).len(), 4); + assert!( + open_shard_ids_of(TID, OLD, &both) + .iter() + .all(|id| id.len() == 61) + ); + // Another table's rows are never selected. + let foreign = vec![ShardRow { + table_id: "other".to_owned(), + ..row( + shard_id("other", OLD, StreamViewType::KeysOnly, 0).unwrap(), + None, + ) + }]; + assert!(open_shard_ids_of(TID, OLD, &foreign).is_empty()); +} + +#[test] +fn an_enable_re_read_sees_whether_its_stream_is_still_current() { + let enabled = table(Some(NEW), Some(spec(true, Some(StreamViewType::NewImage)))); + assert!(enable_still_current(Some(&enabled), NEW)); + // A disable committed between the enable's catalog write and its shard + // inserts: the label is still the enable's, the specification is not + // enabled. + let disabled = table(Some(NEW), Some(spec(false, Some(StreamViewType::NewImage)))); + assert!(!enable_still_current(Some(&disabled), NEW)); + // A later enable replaced the label. + assert!(!enable_still_current(Some(&enabled), OLD)); + // No specification at all, or no table row. + assert!(!enable_still_current(Some(&table(Some(NEW), None)), NEW)); + assert!(!enable_still_current(None, NEW)); +} diff --git a/crates/storage/src/util/arn.rs b/crates/storage/src/util/arn.rs index f463ee1f0..e9a6f4c5c 100755 --- a/crates/storage/src/util/arn.rs +++ b/crates/storage/src/util/arn.rs @@ -119,20 +119,23 @@ pub fn stream_does_not_exist() -> StorageError { } /// What `GetShardIterator` answers for a shard that is not one of the named -/// stream's shards: the stream-not-found sentence with a trailing period. The -/// service answers "Requested resource not found: Shard does not exist" -/// (measured 2026-10-24), so this text differs from it. +/// stream's shards. Measured 2026-10-24 for a shard id of the table's other +/// stream, on the old and on the new stream ARN, and for a well-formed shard id +/// the account never had. +pub const SHARD_DOES_NOT_EXIST: &str = "Requested resource not found: Shard does not exist"; + +/// [`SHARD_DOES_NOT_EXIST`] as the storage error the engine maps to a +/// `ResourceNotFoundException`. #[must_use] -pub fn shard_not_in_stream(stream_arn: &str) -> StorageError { - StorageError::TableNotFound(format!( - "Requested resource not found: Stream: {stream_arn} not found." - )) +pub fn shard_does_not_exist() -> StorageError { + StorageError::TableNotFound(SHARD_DOES_NOT_EXIST.to_owned()) } #[cfg(test)] mod stream_not_found_tests { use super::{ - STREAM_DOES_NOT_EXIST, shard_not_in_stream, stream_does_not_exist, stream_not_found, + SHARD_DOES_NOT_EXIST, STREAM_DOES_NOT_EXIST, shard_does_not_exist, stream_does_not_exist, + stream_not_found, }; use crate::error::StorageError; @@ -164,11 +167,12 @@ mod stream_not_found_tests { } #[test] - fn a_shard_of_no_stream_of_the_table_keeps_the_sentence_with_the_period() { + fn get_shard_iterator_says_a_shard_of_no_stream_of_the_table_does_not_exist() { assert_eq!( - message(shard_not_in_stream(ARN)), - format!("Requested resource not found: Stream: {ARN} not found.") + SHARD_DOES_NOT_EXIST, + "Requested resource not found: Shard does not exist" ); + assert_eq!(message(shard_does_not_exist()), SHARD_DOES_NOT_EXIST); } } diff --git a/crates/storage/src/util/mod.rs b/crates/storage/src/util/mod.rs index bb2493827..a22aec27a 100755 --- a/crates/storage/src/util/mod.rs +++ b/crates/storage/src/util/mod.rs @@ -13,9 +13,9 @@ mod key; mod shard_stream; pub use arn::{ - ARN_PARTITION, STREAM_DOES_NOT_EXIST, format_stream_label, index_arn, new_stream_label, - parse_stream_arn, shard_not_in_stream, stream_arn, stream_does_not_exist, stream_not_found, - table_arn, + ARN_PARTITION, SHARD_DOES_NOT_EXIST, STREAM_DOES_NOT_EXIST, format_stream_label, index_arn, + new_stream_label, parse_stream_arn, shard_does_not_exist, stream_arn, stream_does_not_exist, + stream_not_found, table_arn, }; pub use backup_id::{ BackupArnError, BackupArnParts, MAX_BACKUP_ARN_LEN, backup_arn, backup_creation_date_time, diff --git a/docs/design/07-component-streams.md b/docs/design/07-component-streams.md index dbff306b5..d87708787 100755 --- a/docs/design/07-component-streams.md +++ b/docs/design/07-component-streams.md @@ -91,6 +91,18 @@ DynamoDB retains stream records for 24 hours. Options: **Recommended:** Application-level background task (same pattern as TTL cleanup), configurable retention period (default: 24 hours). +### 4.5 Enable and Disable + +A table has a sequence of streams. `UpdateTable StreamEnabled=false` closes the current one: the stored specification is cleared (DescribeTable omits `StreamSpecification`), `LatestStreamArn` and `LatestStreamLabel` keep naming the closed stream, DescribeStream reports it `DISABLED` with its original view type, and every open shard gets an `EndingSequenceNumber` (its last record's sequence number, or its starting number when it holds none). `StreamEnabled=true` afterwards creates a new stream: a new label later than the previous one, four new shard ids, the requested view type. The closed stream stays describable and listed; ListStreams lists every stream of a table, newest first. Enabling a table whose stream is enabled, and disabling a table with no enabled stream, are refused with `ValidationException`. + +Each stream's identity is carried in its shard ids, `shardId-{table_id}-{label_millis:013}{view_type:1}{index:02}`, so no backend stores a stream table. Shard ids from before this scheme (`shardId-{table_id}-{index}`, and the `shardId-{table_name}-{index}` form SQLite and PostgreSQL wrote before v0.1.14) belong to the table's current stream. Writes record only to open shards. The rules shared by the four backends live in `extenddb_storage::stream_lineage`; each backend reads its table row and shard rows and calls them. + +On Cassandra the write paths compute a record's partition from the table id and partition key alone, inside the data write's LOGGED BATCH, so records stay in one partition per shard index and a stream's shard is a range of that partition: after its `StartingSequenceNumber`, up to its `EndingSequenceNumber` once closed. A new stream's shard starts where the closed one on the same index ended. + +A stream change travels alone: an `UpdateTable` that carries `StreamSpecification` beside `BillingMode` or `ProvisionedThroughput` is refused with `You cannot modify stream status while updating table IOPS`, beside any `GlobalSecondaryIndexUpdates` entry with `You cannot create or delete index while changing stream status`, and nothing is applied. A table created with `StreamSpecification {StreamEnabled: false}` stores nothing of it. + +Writes racing a disable. The service's rule (measured 2026-10-24, 569 writes against one disable): every write accepted before the stream reports `DISABLED` lands in the closed stream before its shard's `EndingSequenceNumber`, and a write after that records nothing. ExtendDB closes the shards in the `UpdateTable` call. On SQLite the engine write lock serialises the close against every write. On PostgreSQL the write path reads the open shards `FOR SHARE` in its data transaction and the close locks them `FOR UPDATE` before taking the ending numbers, in one data transaction, so a write that read the open set commits before the ending number is taken and a later write re-evaluates the closed rows and records nothing. On Cassandra a new stream's shards start at a clock value drawn at enable time, above every sequence issued before, so a write of the closed stream that commits after the close falls between the old shard's ending number and the new shard's starting number and is read by neither stream (the sequence is drawn in the write's LOGGED BATCH, which the close cannot wait for). On MongoDB the window is one write transaction wide: a write whose transaction read the shard as open and commits after the close lands past the ending number; closing the gap needs the write transaction to touch the shard document and the stream path to classify the resulting write conflict as retryable, which it does not today. On every backend a write that found no open shard because its key info pre-dated the disable is acknowledged and recorded in no stream; the service records every write it acknowledges while the disable is in progress. A table toggled many times accumulates four closed shard rows per cycle; nothing sweeps a live table's closed shards. On PostgreSQL, MongoDB and Cassandra an enable commits its catalog row before inserting its shards, so a disable that reads the row in between closes whatever shards exist at that instant; the enable re-reads the row after its inserts and closes its own shards when the row no longer says enabled with its label, so a stream never reports `DISABLED` with open shards. + ## 5. Integration Points ### 5.1 Write Path Integration @@ -158,7 +170,7 @@ GetRecords handler: | Cross-instance stream consistency | Deferred | Relevant for multi-instance deployments | | Stream record deduplication | Deferred | Relevant for at-least-once delivery guarantees | | Kinesis adapter compatibility | Deferred | DynamoDB Streams has a Kinesis-compatible adapter | -| Stream enable/disable lifecycle | Deferred | What happens to in-flight records when streams are disabled | +| Stream enable/disable lifecycle | Decided | See 4.5: disable closes the shards, enable creates a new stream | --- diff --git a/docs/differences-from-dynamodb.md b/docs/differences-from-dynamodb.md index 80c1f671b..8eb49ecee 100755 --- a/docs/differences-from-dynamodb.md +++ b/docs/differences-from-dynamodb.md @@ -48,6 +48,7 @@ adaptation when switching between ExtendDB and the real service. |------|----------|------| | Table creation delay | Returns `CREATING` immediately; transitions to `ACTIVE` typically within seconds. Same behavior for on-demand and provisioned | Configurable via `control_plane_delay_seconds` runtime setting (default: 5s) | | DeletionProtectionEnabled | Enforced | Enforced (accepted and stored, DeleteTable rejects when enabled) | +| Stream disable and enable | `UpdateTable StreamEnabled=false` answers `UPDATING`; the stream reports `DISABLED` a few seconds later with every shard closed, and every write acknowledged before that lands in the closed stream. `StreamEnabled=true` afterwards creates a new stream with a later label; both streams stay listed, and a disabled stream stays describable after its table is deleted | The shards close inside the `UpdateTable` call and the response says `ACTIVE`. A write that reaches storage after the close is acknowledged and recorded in no stream (on MongoDB it is recorded past the shard's `EndingSequenceNumber` instead); the window is one write wide. A disabled stream is removed with its table. A stream created by a release before this one loses its ARN once the table is disabled and re-enabled (its label lived only in the table row) | ## Time to Live (TTL) diff --git a/tests/python/test_stream_disable.py b/tests/python/test_stream_disable.py new file mode 100644 index 000000000..1b7c175e2 --- /dev/null +++ b/tests/python/test_stream_disable.py @@ -0,0 +1,416 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Disabling a table's stream closes it; enabling it again creates a new one. + +One test per row measured on Amazon DynamoDB on 2026-10-24 (us-east-1, client +retries off), on a table with a NEW_AND_OLD_IMAGES stream and three items. The +module fixture runs the whole sequence once, in the order it was measured, and +records every answer; each test asserts one row. + +Runs against ExtendDB (DYNAMODB_ENDPOINT) and against the service (streams +client on its default endpoint). On the service the disable completes +asynchronously, so the fixture waits for DISABLED. +""" + +from __future__ import annotations + +import os +import time + +import boto3 +import botocore.config +import pytest +from botocore.exceptions import ClientError + +from helpers import unique_name, wait_for_active, wait_for_deleted + +NEW_AND_OLD = {"StreamEnabled": True, "StreamViewType": "NEW_AND_OLD_IMAGES"} +ITEMS = ["k0", "k1", "k2"] + + +def _streams_client(endpoint_url): + kwargs: dict = { + "service_name": "dynamodbstreams", + "region_name": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + "config": botocore.config.Config(retries={"max_attempts": 0}), + } + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + ca_cert = os.environ.get("EXTENDDB_CA_CERT", "") + kwargs["verify"] = ca_cert if ca_cert else False + return boto3.client(**kwargs) + + +def _call(fn, **kwargs) -> dict: + """The response without its metadata, or the error's status, code and message.""" + try: + resp = fn(**kwargs) + resp.pop("ResponseMetadata", None) + return {"status": 200, "response": resp} + except ClientError as e: + return { + "status": e.response["ResponseMetadata"]["HTTPStatusCode"], + "code": e.response["Error"]["Code"], + "message": e.response["Error"]["Message"], + } + + +def _stream(streams, arn: str, **kwargs) -> dict: + return streams.describe_stream(StreamArn=arn, **kwargs)["StreamDescription"] + + +def _wait_disabled(streams, arn: str, timeout: float = 120.0) -> dict: + deadline = time.monotonic() + timeout + while True: + desc = _stream(streams, arn) + if desc["StreamStatus"] == "DISABLED" or time.monotonic() > deadline: + return desc + time.sleep(3) + + +def _iterator(streams, arn: str, shard: str, kind: str, seq: str | None = None) -> dict: + kwargs = {"StreamArn": arn, "ShardId": shard, "ShardIteratorType": kind} + if seq is not None: + kwargs["SequenceNumber"] = seq + return _call(streams.get_shard_iterator, **kwargs) + + +def _reread(streams, arn: str, shards: list[str]) -> dict: + """Every record of every shard, read from a fresh TRIM_HORIZON iterator.""" + out = {} + for shard in shards: + it = _iterator(streams, arn, shard, "TRIM_HORIZON")["response"]["ShardIterator"] + got = [] + for _ in range(50): + page = streams.get_records(ShardIterator=it, Limit=1000) + got += [r["dynamodb"]["SequenceNumber"] for r in page["Records"]] + it = page.get("NextShardIterator") + if not it or not page["Records"]: + break + out[shard] = got + return out + + +def _closed_shard_page_ends(page: dict) -> None: + """The page that reaches the end of a closed shard omits NextShardIterator. + + That is PR #372's rule (a closed and drained shard ends the iterator); + until it lands ExtendDB keeps answering an iterator, so on ExtendDB this + one assertion is an expected failure. Every other assertion of the row + stays unconditional. + """ + if "NextShardIterator" not in page["response"]: + return + if os.environ.get("DYNAMODB_ENDPOINT", "").strip(): + pytest.xfail("closed shard terminal GetRecords page: PR #372") + raise AssertionError(f"NextShardIterator present on a closed shard's final page: {page}") + + +def _iterate(streams, arn: str, shard: str, kind: str, seq: str | None = None): + """GetShardIterator, then one GetRecords with the iterator it issued.""" + it = _iterator(streams, arn, shard, kind, seq) + if it["status"] != 200: + return it, None + return it, _call(streams.get_records, ShardIterator=it["response"]["ShardIterator"]) + + +@pytest.fixture(scope="module") +def run(dynamodb_client, endpoint_url): + """Run the measured sequence once and record every answer by row name.""" + ddb = dynamodb_client + streams = _streams_client(endpoint_url) + name = unique_name("t482") + nostream = unique_name("x482-nostream") + key = { + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + } + ddb.create_table(TableName=name, StreamSpecification=NEW_AND_OLD, **key) + ddb.create_table(TableName=nostream, **key) + wait_for_active(ddb, name) + wait_for_active(ddb, nostream) + rows: dict = {"name": name, "nostream": nostream} + try: + table = ddb.describe_table(TableName=name)["Table"] + rows["describe_enabled"] = table + old_arn = table["LatestStreamArn"] + rows["old_arn"] = old_arn + for pk in ITEMS: + ddb.put_item(TableName=name, Item={"pk": {"S": pk}, "v": {"S": pk}}) + shards = [s["ShardId"] for s in _stream(streams, old_arn)["Shards"]] + rows["old_shards"] = shards + rows["records"] = {} + rows["pre_disable_iterators"] = {} + for shard in shards: + it = _iterator(streams, old_arn, shard, "TRIM_HORIZON")["response"]["ShardIterator"] + got = streams.get_records(ShardIterator=it, Limit=1000)["Records"] + rows["records"][shard] = [r["dynamodb"]["SequenceNumber"] for r in got] + rows["pre_disable_iterators"][shard] = _iterator( + streams, old_arn, shard, "TRIM_HORIZON" + )["response"]["ShardIterator"] + + for view in ["NEW_AND_OLD_IMAGES", "KEYS_ONLY"]: + rows[f"enable_while_enabled_{view}"] = _call( + ddb.update_table, + TableName=name, + StreamSpecification={"StreamEnabled": True, "StreamViewType": view}, + ) + rows["disable_never_streamed"] = _call( + ddb.update_table, TableName=nostream, StreamSpecification={"StreamEnabled": False} + ) + + rows["disable"] = _call( + ddb.update_table, TableName=name, StreamSpecification={"StreamEnabled": False} + ) + wait_for_active(ddb, name) + rows["describe_disabled"] = ddb.describe_table(TableName=name)["Table"] + rows["stream_disabled"] = _wait_disabled(streams, old_arn) + rows["disable_while_disabled"] = _call( + ddb.update_table, TableName=name, StreamSpecification={"StreamEnabled": False} + ) + + rows["latest"] = {} + rows["pre_disable_reads"] = {} + rows["after_last"] = {} + rows["at_ending"] = {} + for s in rows["stream_disabled"]["Shards"]: + shard = s["ShardId"] + rows["latest"][shard] = _iterate(streams, old_arn, shard, "LATEST") + rows["pre_disable_reads"][shard] = _call( + streams.get_records, ShardIterator=rows["pre_disable_iterators"][shard] + ) + if rows["records"][shard]: + last = rows["records"][shard][-1] + rows["after_last"][shard] = _iterate( + streams, old_arn, shard, "AFTER_SEQUENCE_NUMBER", last + ) + # Absent on a server that does not close the shard; the row + # for the ending number reports that on its own. + ending = s["SequenceNumberRange"].get("EndingSequenceNumber", last) + rows["at_ending"][shard] = _iterate( + streams, old_arn, shard, "AT_SEQUENCE_NUMBER", ending + ) + + page1 = _stream(streams, old_arn, Limit=2) + page2 = _stream( + streams, old_arn, Limit=2, ExclusiveStartShardId=page1.get("LastEvaluatedShardId", "") + ) + rows["paged"] = (page1, page2) + + # A write while the stream is disabled records nothing (0 of 1 on the + # service); every old shard is re-read from TRIM_HORIZON afterwards. + ddb.put_item(TableName=name, Item={"pk": {"S": "while-disabled"}}) + rows["old_records_after_disabled_write"] = _reread(streams, old_arn, shards) + + rows["reenable"] = _call(ddb.update_table, TableName=name, StreamSpecification=NEW_AND_OLD) + wait_for_active(ddb, name) + rows["describe_reenabled"] = ddb.describe_table(TableName=name)["Table"] + new_arn = rows["describe_reenabled"]["LatestStreamArn"] + rows["new_arn"] = new_arn + rows["stream_new"] = _stream(streams, new_arn) + ddb.put_item(TableName=name, Item={"pk": {"S": "after"}}) + rows["stream_old_after_write"] = _stream(streams, old_arn) + rows["old_records_after_reenable_write"] = _reread(streams, old_arn, shards) + + rows["list_all"] = streams.list_streams()["Streams"] + rows["list_table"] = streams.list_streams(TableName=name)["Streams"] + new_shard = rows["stream_new"]["Shards"][0]["ShardId"] + rows["cross_old_arn"] = _iterator(streams, old_arn, new_shard, "TRIM_HORIZON") + rows["cross_new_arn"] = _iterator(streams, new_arn, shards[0], "TRIM_HORIZON") + yield rows + finally: + for t in (name, nostream): + try: + ddb.delete_table(TableName=t) + wait_for_deleted(ddb, t) + except ClientError: + pass + + +def _refused(row: dict, sentence: str) -> None: + assert (row.get("status"), row.get("code"), row.get("message")) == ( + 400, + "ValidationException", + sentence, + ), row + + +def test_enable_while_enabled_same_view_type_is_refused(run): + _refused( + run["enable_while_enabled_NEW_AND_OLD_IMAGES"], + f"Table already has an enabled stream: TableName: {run['name']}", + ) + + +def test_enable_while_enabled_other_view_type_is_refused(run): + _refused( + run["enable_while_enabled_KEYS_ONLY"], + f"Table already has an enabled stream: TableName: {run['name']}", + ) + + +def test_disable_on_a_table_that_never_had_a_stream_is_refused(run): + _refused( + run["disable_never_streamed"], + f"Table has no stream to disable: TableName: {run['nostream']}", + ) + + +def test_disable_while_disabled_is_refused(run): + _refused( + run["disable_while_disabled"], + f"Table has no stream to disable: TableName: {run['name']}", + ) + + +def test_disable_response_omits_the_specification_and_keeps_the_latest_stream(run): + assert run["disable"]["status"] == 200, run["disable"] + desc = run["disable"]["response"]["TableDescription"] + # The service answers TableStatus UPDATING here; ExtendDB applies every + # UpdateTable change at once and answers ACTIVE, for stream changes as for + # billing-mode changes, so the status is not asserted. + assert "StreamSpecification" not in desc + enabled = run["describe_enabled"] + assert desc["LatestStreamArn"] == enabled["LatestStreamArn"] + assert desc["LatestStreamLabel"] == enabled["LatestStreamLabel"] + + +def test_reenable_response_names_a_new_stream(run): + assert run["reenable"]["status"] == 200, run["reenable"] + desc = run["reenable"]["response"]["TableDescription"] + assert desc["StreamSpecification"] == NEW_AND_OLD + assert desc["LatestStreamArn"] != run["old_arn"] + assert desc["LatestStreamLabel"] > run["describe_enabled"]["LatestStreamLabel"] + + +def test_describe_table_while_enabled(run): + t = run["describe_enabled"] + assert t["StreamSpecification"] == NEW_AND_OLD + assert t["LatestStreamArn"].endswith("/stream/" + t["LatestStreamLabel"]) + + +def test_describe_table_while_disabled(run): + t = run["describe_disabled"] + assert t["TableStatus"] == "ACTIVE" + assert "StreamSpecification" not in t + assert t["LatestStreamArn"] == run["old_arn"] + assert t["LatestStreamLabel"] == run["describe_enabled"]["LatestStreamLabel"] + + +def test_describe_table_after_reenable(run): + t = run["describe_reenabled"] + assert t["StreamSpecification"] == NEW_AND_OLD + assert t["LatestStreamArn"] == run["new_arn"] != run["old_arn"] + assert run["new_arn"].endswith("/stream/" + t["LatestStreamLabel"]) + + +def test_describe_stream_of_a_disabled_stream(run): + d = run["stream_disabled"] + assert d["StreamStatus"] == "DISABLED" + assert d["StreamViewType"] == "NEW_AND_OLD_IMAGES" + assert d["StreamArn"] == run["old_arn"] + assert d["TableName"] == run["name"] + assert d["KeySchema"] == [{"AttributeName": "pk", "KeyType": "HASH"}] + assert "LastEvaluatedShardId" not in d + assert [s["ShardId"] for s in d["Shards"]] == run["old_shards"] + assert all("ParentShardId" not in s for s in d["Shards"]) + + +def test_every_closed_shard_carries_its_ending_sequence_number(run): + for s in run["stream_disabled"]["Shards"]: + r = s["SequenceNumberRange"] + written = run["records"][s["ShardId"]] + # A shard with records ends at its last record; an empty one ends at + # its starting number. + expected = written[-1] if written else r["StartingSequenceNumber"] + assert r.get("EndingSequenceNumber") == expected, (s, written) + assert sum(len(v) for v in run["records"].values()) == len(ITEMS) + + +def test_describe_stream_pages_a_disabled_stream(run): + page1, page2 = run["paged"] + assert [s["ShardId"] for s in page1["Shards"]] == run["old_shards"][:2] + assert page1["LastEvaluatedShardId"] == run["old_shards"][1] + assert [s["ShardId"] for s in page2["Shards"]] == run["old_shards"][2:] + assert "LastEvaluatedShardId" not in page2 + + +def test_latest_on_a_closed_shard_reads_an_empty_final_page(run): + for shard, (it, page) in run["latest"].items(): + assert it["status"] == 200, (shard, it) + assert page["status"] == 200, (shard, page) + assert page["response"]["Records"] == [], (shard, page) + _closed_shard_page_ends(page) + + +def test_an_iterator_from_before_the_disable_reads_to_the_end(run): + for shard, page in run["pre_disable_reads"].items(): + assert page["status"] == 200, (shard, page) + got = [r["dynamodb"]["SequenceNumber"] for r in page["response"]["Records"]] + assert got == run["records"][shard], (shard, page) + _closed_shard_page_ends(page) + + +def test_after_the_last_record_of_a_closed_shard_is_an_empty_final_page(run): + assert run["after_last"], "no shard held a record" + for shard, (it, page) in run["after_last"].items(): + assert it["status"] == 200, (shard, it) + assert page["status"] == 200, (shard, page) + assert page["response"]["Records"] == [], (shard, page) + _closed_shard_page_ends(page) + + +def test_at_the_ending_number_reads_the_last_record_and_ends(run): + assert run["at_ending"], "no shard held a record" + for shard, (it, page) in run["at_ending"].items(): + assert it["status"] == 200, (shard, it) + assert page["status"] == 200, (shard, page) + got = [r["dynamodb"]["SequenceNumber"] for r in page["response"]["Records"]] + assert got == run["records"][shard][-1:], (shard, page) + _closed_shard_page_ends(page) + + +def test_a_write_while_the_stream_is_disabled_records_nothing(run): + assert run["old_records_after_disabled_write"] == run["records"] + + +def test_reenable_creates_a_new_stream_with_new_shards(run): + new = run["stream_new"] + assert new["StreamStatus"] == "ENABLED" + assert new["StreamArn"] == run["new_arn"] + assert new["StreamLabel"] > run["describe_enabled"]["LatestStreamLabel"] + ids = [s["ShardId"] for s in new["Shards"]] + assert ids and not set(ids) & set(run["old_shards"]) + assert all("EndingSequenceNumber" not in s["SequenceNumberRange"] for s in new["Shards"]) + + +def test_the_old_stream_is_unchanged_after_reenable_and_a_write(run): + before, after = run["stream_disabled"], run["stream_old_after_write"] + assert after["StreamStatus"] == "DISABLED" + assert after["Shards"] == before["Shards"] + # The records of every old shard, read afresh, are the pre-disable set. + assert run["old_records_after_reenable_write"] == run["records"] + + +def test_list_streams_lists_both_streams_newest_first(run): + expected = [ + {"StreamArn": arn, "TableName": run["name"], "StreamLabel": arn.rsplit("/", 1)[1]} + for arn in (run["new_arn"], run["old_arn"]) + ] + assert run["list_table"] == expected + # The account-wide call is paged at 100 streams; only the table's own ARNs + # are asserted, when the page reached them. + own = [s for s in run["list_all"] if s["TableName"] == run["name"]] + assert own in (expected, []), own + + +def test_a_shard_of_the_other_stream_does_not_exist(run): + for row in (run["cross_old_arn"], run["cross_new_arn"]): + assert row == { + "status": 400, + "code": "ResourceNotFoundException", + "message": "Requested resource not found: Shard does not exist", + }, row diff --git a/tests/python/test_stream_disable_fold.py b/tests/python/test_stream_disable_fold.py new file mode 100644 index 000000000..9620a0954 --- /dev/null +++ b/tests/python/test_stream_disable_fold.py @@ -0,0 +1,329 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Stream lifecycle rows measured on Amazon DynamoDB on 2026-10-24, second round. + +A stream change beside a billing, throughput or index change; ListStreams +paging over a table with two streams; GetShardIterator on an unknown shard or +stream; a table created with StreamSpecification {StreamEnabled: false}. +Companion to test_stream_disable.py. One module fixture runs the measured +sequence once; each test asserts one row. +""" + +from __future__ import annotations + +import os +import time + +import boto3 +import botocore.config +import pytest +from botocore.exceptions import ClientError + +from helpers import unique_name, wait_for_active, wait_for_deleted + +KEYS_ONLY = {"StreamEnabled": True, "StreamViewType": "KEYS_ONLY"} +NEW_IMAGE = {"StreamEnabled": True, "StreamViewType": "NEW_IMAGE"} +DISABLED = {"StreamEnabled": False} +INVALID = "One or more parameter values were invalid: " + +ON_SERVICE = not os.environ.get("DYNAMODB_ENDPOINT", "").strip() + + +def _streams_client(endpoint_url): + kwargs: dict = { + "service_name": "dynamodbstreams", + "region_name": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + # Client-side parameter validation off: the short ShardId row measures + # the server's own length check. + "config": botocore.config.Config( + retries={"max_attempts": 0}, parameter_validation=False + ), + } + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + ca_cert = os.environ.get("EXTENDDB_CA_CERT", "") + kwargs["verify"] = ca_cert if ca_cert else False + return boto3.client(**kwargs) + + +def _call(fn, **kwargs) -> dict: + try: + resp = fn(**kwargs) + resp.pop("ResponseMetadata", None) + return {"status": 200, "response": resp} + except ClientError as e: + return { + "status": e.response["ResponseMetadata"]["HTTPStatusCode"], + "code": e.response["Error"]["Code"], + "message": e.response["Error"]["Message"], + } + + +def _list(streams, **kwargs) -> dict: + # The service rate-limits ListStreams within a burst. + if ON_SERVICE: + time.sleep(6) + return _call(streams.list_streams, **kwargs) + + +def _wait_disabled(streams, arn: str, timeout: float = 120.0) -> None: + deadline = time.monotonic() + timeout + while streams.describe_stream(StreamArn=arn)["StreamDescription"]["StreamStatus"] != "DISABLED": + if time.monotonic() > deadline: + return + time.sleep(3) + + +def _set_stream(ddb, name: str, spec: dict, **extra) -> dict: + row = _call(ddb.update_table, TableName=name, StreamSpecification=spec, **extra) + wait_for_active(ddb, name) + return row + + +@pytest.fixture(scope="module") +def run(dynamodb_client, endpoint_url): + ddb = dynamodb_client + streams = _streams_client(endpoint_url) + key = { + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + } + a, b, off = unique_name("f482-a"), unique_name("f482-b"), unique_name("f482-off") + ddb.create_table(TableName=a, StreamSpecification=KEYS_ONLY, **key) + ddb.create_table(TableName=b, StreamSpecification=KEYS_ONLY, **key) + ddb.create_table(TableName=off, StreamSpecification=DISABLED, **key) + for t in (a, b, off): + wait_for_active(ddb, t) + rows: dict = {"a": a, "b": b, "off": off} + try: + # F1: a stream change beside another change, while enabled. + rows["disable_plus_billing"] = _set_stream( + ddb, + a, + DISABLED, + BillingMode="PROVISIONED", + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + gsi = { + "IndexName": "gsi1", + "KeySchema": [{"AttributeName": "g", "KeyType": "HASH"}], + "Projection": {"ProjectionType": "ALL"}, + } + rows["disable_plus_gsi"] = _set_stream( + ddb, + a, + DISABLED, + AttributeDefinitions=[{"AttributeName": "g", "AttributeType": "S"}], + GlobalSecondaryIndexUpdates=[{"Create": gsi}], + ) + rows["disable_plus_gsi_undefined_attribute"] = _set_stream( + ddb, + a, + DISABLED, + GlobalSecondaryIndexUpdates=[{"Create": {**gsi, "IndexName": "gsi2"}}], + ) + rows["enable_plus_billing_while_enabled"] = _set_stream( + ddb, a, KEYS_ONLY, BillingMode="PAY_PER_REQUEST" + ) + rows["a_after_refusals"] = ddb.describe_table(TableName=a)["Table"] + + # F3: GetShardIterator on unknown shard or stream. + a_arn = rows["a_after_refusals"]["LatestStreamArn"] + a_shard = streams.describe_stream(StreamArn=a_arn)["StreamDescription"]["Shards"][0]["ShardId"] + rows["unknown_shard"] = _call( + streams.get_shard_iterator, + StreamArn=a_arn, + ShardId="shardId-00000001700000000000-00000000", + ShardIteratorType="TRIM_HORIZON", + ) + rows["short_shard"] = _call( + streams.get_shard_iterator, + StreamArn=a_arn, + ShardId="nope", + ShardIteratorType="TRIM_HORIZON", + ) + unknown_arn = a_arn[:-3] + ("000" if not a_arn.endswith("000") else "001") + rows["unknown_arn"] = unknown_arn + rows["unknown_stream"] = _call( + streams.get_shard_iterator, + StreamArn=unknown_arn, + ShardId=a_shard, + ShardIteratorType="TRIM_HORIZON", + ) + + # F2: two streams on A (and on B), then ListStreams paging. + _set_stream(ddb, a, DISABLED) + _wait_disabled(streams, a_arn) + _set_stream(ddb, a, NEW_IMAGE) + new_arn = ddb.describe_table(TableName=a)["Table"]["LatestStreamArn"] + rows["a_old"], rows["a_new"] = a_arn, new_arn + b_arn = ddb.describe_table(TableName=b)["Table"]["LatestStreamArn"] + rows["list_a"] = _list(streams, TableName=a) + rows["list_a_page1"] = _list(streams, TableName=a, Limit=1) + rows["list_a_page2"] = _list(streams, TableName=a, Limit=1, ExclusiveStartStreamArn=new_arn) + rows["list_a_page3"] = _list(streams, TableName=a, Limit=1, ExclusiveStartStreamArn=a_arn) + rows["list_a_from_older"] = _list(streams, TableName=a, ExclusiveStartStreamArn=a_arn) + rows["list_a_from_newest"] = _list(streams, TableName=a, ExclusiveStartStreamArn=new_arn) + rows["b_arn"] = b_arn + rows["list_a_from_b"] = _list(streams, TableName=a, ExclusiveStartStreamArn=b_arn) + + # F4: the table created with a disabled specification. + rows["off_describe"] = ddb.describe_table(TableName=off)["Table"] + rows["off_disable"] = _set_stream(ddb, off, DISABLED) + rows["off_enable"] = _set_stream(ddb, off, KEYS_ONLY) + rows["off_after_enable"] = ddb.describe_table(TableName=off)["Table"] + yield rows + finally: + for t in (a, b, off): + try: + ddb.delete_table(TableName=t) + wait_for_deleted(ddb, t) + except ClientError: + pass + + +def _refused(row: dict, code: str, sentence: str) -> None: + assert (row.get("status"), row.get("code"), row.get("message")) == (400, code, sentence), row + + +# F1 + + +def test_a_stream_change_beside_a_billing_change_is_refused(run): + _refused( + run["disable_plus_billing"], + "ValidationException", + INVALID + "You cannot modify stream status while updating table IOPS", + ) + + +def test_a_stream_change_beside_an_index_create_is_refused(run): + _refused( + run["disable_plus_gsi"], + "ValidationException", + INVALID + "You cannot create or delete index while changing stream status", + ) + + +def test_the_index_definition_check_precedes_the_companion_rule(run): + row = run["disable_plus_gsi_undefined_attribute"] + assert (row["status"], row["code"]) == (400, "ValidationException"), row + # The index definition check comes first. The service's sentence is + # "AttributeDefinitions is not specified for index: gsi2"; ExtendDB's own + # sentence for an undefined index key ("Some index key attributes are not + # defined in AttributeDefinitions. Keys: [g], ...") pre-dates this change, + # so only the ordering is asserted here. + assert row["message"].startswith(INVALID), row + assert "AttributeDefinitions" in row["message"] and "stream" not in row["message"], row + + +def test_the_already_enabled_check_precedes_the_companion_rule(run): + _refused( + run["enable_plus_billing_while_enabled"], + "ValidationException", + f"Table already has an enabled stream: TableName: {run['a']}", + ) + + +def test_a_refused_combined_update_changes_nothing(run): + t = run["a_after_refusals"] + assert t["StreamSpecification"] == KEYS_ONLY + assert t.get("BillingModeSummary", {}).get("BillingMode", "PAY_PER_REQUEST") == "PAY_PER_REQUEST" + assert "GlobalSecondaryIndexes" not in t + + +# F2 + + +def test_list_streams_of_a_table_with_two_streams(run): + row = run["list_a"] + assert row["status"] == 200, row + assert [s["StreamArn"] for s in row["response"]["Streams"]] == [run["a_new"], run["a_old"]] + assert "LastEvaluatedStreamArn" not in row["response"] + + +def test_list_streams_limit_one_carries_the_last_arn_on_every_cut_page(run): + p1, p2, p3 = run["list_a_page1"], run["list_a_page2"], run["list_a_page3"] + assert [s["StreamArn"] for s in p1["response"]["Streams"]] == [run["a_new"]], p1 + assert p1["response"]["LastEvaluatedStreamArn"] == run["a_new"] + assert [s["StreamArn"] for s in p2["response"]["Streams"]] == [run["a_old"]], p2 + assert p2["response"]["LastEvaluatedStreamArn"] == run["a_old"] + assert p3 == {"status": 200, "response": {"Streams": []}}, p3 + + +def test_list_streams_after_the_older_stream_is_empty(run): + assert run["list_a_from_older"] == {"status": 200, "response": {"Streams": []}} + + +def test_list_streams_after_the_newest_stream_is_the_older_one(run): + row = run["list_a_from_newest"] + assert [s["StreamArn"] for s in row["response"]["Streams"]] == [run["a_old"]], row + assert "LastEvaluatedStreamArn" not in row["response"] + + +def test_list_streams_start_arn_of_another_table_is_refused(run): + _refused( + run["list_a_from_b"], + "ValidationException", + INVALID + f"ExclusiveStartStreamArn provided is invalid: {run['b_arn']}", + ) + + +# F3 + + +def test_an_unknown_shard_id_on_an_existing_stream_does_not_exist(run): + _refused( + run["unknown_shard"], + "ResourceNotFoundException", + "Requested resource not found: Shard does not exist", + ) + + +def test_a_short_shard_id_fails_the_length_constraint(run): + _refused( + run["short_shard"], + "ValidationException", + "1 validation error detected: Value at 'shardId' failed to satisfy constraint: " + "Member must have length greater than or equal to 28", + ) + + +def test_an_unknown_stream_arn_does_not_exist(run): + _refused( + run["unknown_stream"], + "ResourceNotFoundException", + "Requested resource not found: Stream does not exist", + ) + + +# F4 + + +def test_a_table_created_with_a_disabled_specification_stores_none(run): + t = run["off_describe"] + assert "StreamSpecification" not in t + assert "LatestStreamArn" not in t + assert "LatestStreamLabel" not in t + + +def test_disabling_it_is_refused_like_a_table_without_the_member(run): + _refused( + run["off_disable"], + "ValidationException", + f"Table has no stream to disable: TableName: {run['off']}", + ) + + +def test_enabling_it_creates_the_first_stream(run): + row = run["off_enable"] + assert row["status"] == 200, row + desc = row["response"]["TableDescription"] + assert desc["StreamSpecification"] == KEYS_ONLY + assert desc["LatestStreamArn"].endswith("/stream/" + desc["LatestStreamLabel"]) + after = run["off_after_enable"] + assert after["StreamSpecification"] == KEYS_ONLY + assert after["LatestStreamArn"] == desc["LatestStreamArn"] diff --git a/tests/python/test_stream_disable_validation.py b/tests/python/test_stream_disable_validation.py new file mode 100644 index 000000000..2eb1b9acf --- /dev/null +++ b/tests/python/test_stream_disable_validation.py @@ -0,0 +1,272 @@ +# Copyright 2026 ExtendDB contributors +# SPDX-License-Identifier: Apache-2.0 + +"""Stream lifecycle rows measured on Amazon DynamoDB on 2026-10-24, third round. + +Request validation on CreateTable, UpdateTable and the streams API: a stream +enabled without a view type, a specification without StreamEnabled, a disable +beside a view type or beside DeletionProtection, the ShardId upper bound, the +SequenceNumber rules, DescribeStream and ListStreams member constraints, and +the DescribeStream sentence for an unknown stream of an existing table. +Companion to test_stream_disable.py and test_stream_disable_fold.py. +""" + +from __future__ import annotations + +import os + +import boto3 +import botocore.config +import pytest +from botocore.exceptions import ClientError + +from helpers import unique_name, wait_for_active, wait_for_deleted + +KEYS_ONLY = {"StreamEnabled": True, "StreamViewType": "KEYS_ONLY"} +INVALID = "One or more parameter values were invalid: " +VALIDATION = "1 validation error detected: Value " + + +def _client(service: str, endpoint_url): + kwargs: dict = { + "service_name": service, + "region_name": os.environ.get("AWS_DEFAULT_REGION", "us-east-1"), + # Client-side parameter validation off: every row here measures the + # server's own checks. + "config": botocore.config.Config( + retries={"max_attempts": 0}, parameter_validation=False + ), + } + if endpoint_url: + kwargs["endpoint_url"] = endpoint_url + if endpoint_url.startswith("https://"): + ca_cert = os.environ.get("EXTENDDB_CA_CERT", "") + kwargs["verify"] = ca_cert if ca_cert else False + return boto3.client(**kwargs) + + +def _call(fn, **kwargs) -> dict: + try: + resp = fn(**kwargs) + resp.pop("ResponseMetadata", None) + return {"status": 200, "response": resp} + except ClientError as e: + return { + "status": e.response["ResponseMetadata"]["HTTPStatusCode"], + "code": e.response["Error"]["Code"], + "message": e.response["Error"]["Message"], + } + + +def _refused(row: dict, code: str, sentence: str) -> None: + assert (row.get("status"), row.get("code"), row.get("message")) == (400, code, sentence), row + + +@pytest.fixture(scope="module") +def run(endpoint_url): + ddb = _client("dynamodb", endpoint_url) + streams = _client("dynamodbstreams", endpoint_url) + key = { + "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}], + "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}], + "BillingMode": "PAY_PER_REQUEST", + } + on, off = unique_name("v482-on"), unique_name("v482-off") + rows: dict = {"on": on, "off": off} + created = [] + try: + # CreateTable request validation. + never = unique_name("v482-never") + rows["create_enabled_no_view"] = _call( + ddb.create_table, TableName=never, StreamSpecification={"StreamEnabled": True}, **key + ) + rows["create_never_exists"] = _call(ddb.describe_table, TableName=never) + rows["create_no_enabled"] = _call( + ddb.create_table, TableName=never, StreamSpecification={"StreamViewType": "KEYS_ONLY"}, **key + ) + ddb.create_table(TableName=on, StreamSpecification=KEYS_ONLY, **key) + ddb.create_table(TableName=off, **key) + created += [on, off] + wait_for_active(ddb, on) + wait_for_active(ddb, off) + + # UpdateTable request validation, on both tables. + for label, name in (("on", on), ("off", off)): + rows[f"update_no_enabled_{label}"] = _call( + ddb.update_table, TableName=name, StreamSpecification={} + ) + rows[f"disable_with_view_{label}"] = _call( + ddb.update_table, + TableName=name, + StreamSpecification={"StreamEnabled": False, "StreamViewType": "KEYS_ONLY"}, + ) + rows["disable_plus_deletion_protection"] = _call( + ddb.update_table, + TableName=on, + StreamSpecification={"StreamEnabled": False}, + DeletionProtectionEnabled=True, + ) + rows["on_after"] = ddb.describe_table(TableName=on)["Table"] + + # Streams API member constraints. + arn = rows["on_after"]["LatestStreamArn"] + shard = streams.describe_stream(StreamArn=arn)["StreamDescription"]["Shards"][0]["ShardId"] + rows["shard_66"] = _call( + streams.get_shard_iterator, + StreamArn=arn, ShardId="s" * 66, ShardIteratorType="TRIM_HORIZON", + ) + rows["shard_65_unknown"] = _call( + streams.get_shard_iterator, + StreamArn=arn, ShardId="shardId-" + "0" * 57, ShardIteratorType="TRIM_HORIZON", + ) + rows["at_no_sequence"] = _call( + streams.get_shard_iterator, + StreamArn=arn, ShardId=shard, ShardIteratorType="AT_SEQUENCE_NUMBER", + ) + rows["at_sequence_0"] = _call( + streams.get_shard_iterator, + StreamArn=arn, ShardId=shard, ShardIteratorType="AT_SEQUENCE_NUMBER", SequenceNumber="0", + ) + unknown = arn[:-3] + ("000" if not arn.endswith("000") else "001") + rows["unknown_arn"] = unknown + rows["describe_unknown"] = _call(streams.describe_stream, StreamArn=unknown) + rows["describe_short_start"] = _call( + streams.describe_stream, StreamArn=arn, ExclusiveStartShardId="short" + ) + rows["list_limit_0"] = _call(streams.list_streams, TableName=on, Limit=0) + rows["list_short_start"] = _call( + streams.list_streams, TableName=on, ExclusiveStartStreamArn="arn:aws:dynamodb:x" + ) + rows["list_never_existed"] = _call(streams.list_streams, TableName=unique_name("v482-none")) + yield rows + finally: + for t in created: + try: + ddb.delete_table(TableName=t) + wait_for_deleted(ddb, t) + except ClientError: + pass + + +# CreateTable + + +def test_create_table_with_a_stream_enabled_and_no_view_type_is_refused(run): + _refused( + run["create_enabled_no_view"], + "ValidationException", + INVALID + "Table is being created with a stream enabled, UpdateViewType is required", + ) + assert run["create_never_exists"]["code"] == "ResourceNotFoundException", "no table created" + + +def test_create_table_without_stream_enabled_is_refused(run): + _refused( + run["create_no_enabled"], + "ValidationException", + VALIDATION + "null at 'streamSpecification.streamEnabled' failed to satisfy constraint: " + "Member must not be null", + ) + + +# UpdateTable + + +def test_update_table_without_stream_enabled_is_refused(run): + for label in ("on", "off"): + _refused( + run[f"update_no_enabled_{label}"], + "ValidationException", + VALIDATION + "null at 'streamSpecification.streamEnabled' failed to satisfy constraint: " + "Member must not be null", + ) + + +def test_a_disable_beside_a_view_type_is_refused_before_the_no_stream_rule(run): + for label in ("on", "off"): + _refused( + run[f"disable_with_view_{label}"], + "ValidationException", + INVALID + "If stream is being disabled, then UpdateViewType must not be specified", + ) + + +def test_a_disable_beside_deletion_protection_is_refused(run): + _refused( + run["disable_plus_deletion_protection"], + "ValidationException", + INVALID + "DeletionProtection modification must be the only operation in the request", + ) + t = run["on_after"] + assert t["StreamSpecification"] == KEYS_ONLY, "nothing applied" + assert t.get("DeletionProtectionEnabled", False) is False + + +# Streams API + + +def test_a_shard_id_over_65_characters_fails_the_length_constraint(run): + _refused( + run["shard_66"], + "ValidationException", + VALIDATION + "at 'shardId' failed to satisfy constraint: Member must have length less than or equal to 65", + ) + + +def test_a_65_character_unknown_shard_id_does_not_exist(run): + _refused( + run["shard_65_unknown"], + "ResourceNotFoundException", + "Requested resource not found: Shard does not exist", + ) + + +def test_at_sequence_number_without_a_sequence_number_is_refused(run): + _refused(run["at_no_sequence"], "ValidationException", INVALID + "SequenceNumber cannot be null") + + +def test_a_sequence_number_under_21_characters_fails_the_length_constraint(run): + _refused( + run["at_sequence_0"], + "ValidationException", + VALIDATION + "at 'sequenceNumber' failed to satisfy constraint: " + "Member must have length greater than or equal to 21", + ) + + +def test_describe_stream_of_an_unknown_stream_of_an_existing_table(run): + _refused( + run["describe_unknown"], + "ResourceNotFoundException", + f"Requested resource not found: Stream: {run['unknown_arn']} not found", + ) + + +def test_describe_stream_with_a_short_exclusive_start_shard_id_is_refused(run): + _refused( + run["describe_short_start"], + "ValidationException", + VALIDATION + "at 'exclusiveStartShardId' failed to satisfy constraint: " + "Member must have length greater than or equal to 28", + ) + + +def test_list_streams_limit_zero_is_refused(run): + _refused( + run["list_limit_0"], + "ValidationException", + VALIDATION + "at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1", + ) + + +def test_list_streams_with_a_short_start_arn_is_refused(run): + _refused( + run["list_short_start"], + "ValidationException", + VALIDATION + "at 'exclusiveStartStreamArn' failed to satisfy constraint: " + "Member must have length greater than or equal to 37", + ) + + +def test_list_streams_of_a_table_that_never_existed_is_empty(run): + assert run["list_never_existed"] == {"status": 200, "response": {"Streams": []}} From 6f9089f706f050128f9205d52f7ee78d3b9cf018 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:18:18 +0000 Subject: [PATCH 02/15] fix(streams): GetRecords resolves a shard against the stream its id names util::shard_in_stream decided ownership by the table's current label alone, which was right while a table had one stream. With a stream per enable, that rule made a closed stream unreadable as soon as the table was enabled again (its ARN answered "Stream does not exist") and let an iterator pairing the new ARN with an old shard id read the old stream. The rule now reads the shard id with stream_lineage::parse_shard_id: a labelled id belongs to the label it carries, and an id from before streams were named in shard ids belongs to the table's current stream while the table has no labelled shard. From a table's first enable under this release every later stream is labelled, so legacy rows beside a labelled row are the earlier, closed stream, which is how DescribeStream and GetShardIterator already group them. All four backends call it from get_stream_records, so a closed stream stays readable under its own ARN after the table has a newer one, and a token pairing the newer ARN with an older stream's shard id, labelled or legacy, is refused. Whether the table has a labelled shard is read only for a legacy id, so a labelled id keeps #543's statements: an EXISTS on stream_shards over the byte-order range of the table's labelled ids on SQLite and PostgreSQL (stream_lineage::labelled_shard_id_range; COLLATE "C" on PostgreSQL), a find_one on the {table_id, shard_id} index with the labelled id pattern on MongoDB, whose legacy ids have 12-digit suffixes, and the table's shard rows on Cassandra. Unit tests on the rule (a legacy shard under a table that has a labelled stream for the current label is not its stream; a labelled shard is decided without reading the table's shards) and on the labelled range, and SQLite engine tests reading a re-enabled table: the closed stream under its own ARN, the new stream under its, the two cross pairings refused, and the new ARN with a legacy shard of a closed stream refused. --- crates/storage-cassandra/src/stream_engine.rs | 35 +++- crates/storage-mongodb/src/stream_engine.rs | 30 ++- crates/storage-postgres/src/stream_engine.rs | 28 ++- crates/storage-sqlite/src/stream.rs | 29 ++- crates/storage-sqlite/src/stream_tests.rs | 173 ++++++++++++++++ crates/storage/src/lib.rs | 6 +- crates/storage/src/stream_lineage.rs | 36 ++++ crates/storage/src/stream_lineage_tests.rs | 38 ++++ crates/storage/src/util/shard_stream.rs | 184 ++++++++++++++++-- 9 files changed, 521 insertions(+), 38 deletions(-) diff --git a/crates/storage-cassandra/src/stream_engine.rs b/crates/storage-cassandra/src/stream_engine.rs index d1d551aaa..81e0ac8de 100755 --- a/crates/storage-cassandra/src/stream_engine.rs +++ b/crates/storage-cassandra/src/stream_engine.rs @@ -99,7 +99,8 @@ impl StreamEngine for CassandraEngine { // row and the records, issued together and awaited together on the // account keyspace. The shard is the table's when its row carries // that table id, and the stream's when `shard_in_stream` says so - // from the table's label and the shard id; otherwise (another + // from the table's label and the shard id (with a read of the + // table's shard rows for a legacy id only); otherwise (another // table, another account, or a stream of the same table the shard // is not part of) the answer is the "stream does not exist" error // and the records read are discarded. The keyspace follows from the @@ -167,15 +168,29 @@ impl StreamEngine for CassandraEngine { .as_ref() .and_then(|row| row.get_r_by_name("table_id").ok()); let range = match shard_row { - Some(row) - if shard_table_id.as_deref() == Some(table.table_id.as_str()) - && shard_in_stream( - &table.table_id, - &shard_id, - table.stream_label.as_deref(), - &stream_label, - ) => - { + Some(row) if shard_table_id.as_deref() == Some(table.table_id.as_str()) => { + // A legacy id needs the table's shard rows: it is an + // earlier, closed stream once the table has a labelled + // shard. + let owned = shard_in_stream( + &table.table_id, + &shard_id, + table.stream_label.as_deref(), + &stream_label, + || async { + let rows = self + .stream_shard_rows(&keyspace, Some(&table.table_id)) + .await?; + Ok::(stream_lineage::has_labelled_shard( + &table.table_id, + &rows, + )) + }, + ) + .await?; + if !owned { + return Err(extenddb_storage::util::stream_does_not_exist()); + } shard_range_of(physical, &row) } _ => return Err(extenddb_storage::util::stream_does_not_exist()), diff --git a/crates/storage-mongodb/src/stream_engine.rs b/crates/storage-mongodb/src/stream_engine.rs index b779d1fe4..4d5d7b826 100644 --- a/crates/storage-mongodb/src/stream_engine.rs +++ b/crates/storage-mongodb/src/stream_engine.rs @@ -248,7 +248,9 @@ impl StreamEngine for MongoEngine { // shard that is not that table's (another table, or another // account) returns no document. Whether the shard belongs to the // stream the ARN's label names is decided in Rust from the table's - // label and the shard id (`shard_in_stream`). + // label and the shard id (`shard_in_stream`), with a third round + // trip for a legacy id only: whether the table has a labelled + // shard. let table_doc = self .catalog_db .collection::("tables") @@ -308,7 +310,11 @@ impl StreamEngine for MongoEngine { let Some(row_shard_id) = docs.first().and_then(|d| d.get_str("shard_id").ok()) else { return Err(extenddb_storage::util::stream_does_not_exist()); }; - if !shard_in_stream(&table_id, row_shard_id, table_label, &stream_label) { + if !shard_in_stream(&table_id, row_shard_id, table_label, &stream_label, || { + self.table_has_labelled_shard(&table_id) + }) + .await? + { return Err(extenddb_storage::util::stream_does_not_exist()); } @@ -598,6 +604,26 @@ mod tests { /// The records in the result of the `get_stream_records` aggregation; see the /// module documentation. +impl MongoEngine { + /// Whether table `table_id` has a labelled shard document + /// (`stream_lineage::labelled_shard_id_pattern`, on the + /// `{table_id, shard_id}` index), which makes its legacy shards an + /// earlier, closed stream (`util::shard_in_stream`). + async fn table_has_labelled_shard(&self, table_id: &str) -> Result { + let found = self + .data_db + .collection::("stream_shards") + .find_one(doc! { + "table_id": table_id, + "shard_id": { "$regex": stream_lineage::labelled_shard_id_pattern(table_id) }, + }) + .projection(doc! { "_id": 0, "shard_id": 1 }) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(found.is_some()) + } +} + #[path = "stream_record_docs.rs"] mod record_docs; diff --git a/crates/storage-postgres/src/stream_engine.rs b/crates/storage-postgres/src/stream_engine.rs index 9e5419d52..c9af321af 100755 --- a/crates/storage-postgres/src/stream_engine.rs +++ b/crates/storage-postgres/src/stream_engine.rs @@ -146,7 +146,8 @@ impl StreamEngine for PostgresEngine { // shard that is not that table's (another table, or another // account) returns no rows. Whether the shard belongs to the stream // the ARN's label names is decided in Rust from the table's label - // and the shard id (`shard_in_stream`). + // and the shard id (`shard_in_stream`), with a third round trip for + // a legacy id only: whether the table has a labelled shard. let table: Option<(String, Option)> = sqlx::query_as( "SELECT table_id, stream_label FROM tables \ WHERE account_id = $1 AND table_name = $2", @@ -187,7 +188,10 @@ impl StreamEngine for PostgresEngine { row_shard_id, table_label.as_deref(), &stream_label, - ) { + || self.table_has_labelled_shard(&table_id), + ) + .await? + { return Err(extenddb_storage::util::stream_does_not_exist()); } @@ -384,3 +388,23 @@ impl StreamEngine for PostgresEngine { }) } } + +impl PostgresEngine { + /// Whether table `table_id` has a labelled shard row + /// (`stream_lineage::labelled_shard_id_range`, compared in byte order + /// whatever the database's collation), which makes its legacy rows an + /// earlier, closed stream (`util::shard_in_stream`). + async fn table_has_labelled_shard(&self, table_id: &str) -> Result { + let (from, to) = extenddb_storage::stream_lineage::labelled_shard_id_range(table_id); + sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM stream_shards WHERE table_id = $1 \ + AND shard_id COLLATE \"C\" >= $2 AND shard_id COLLATE \"C\" < $3)", + ) + .bind(table_id) + .bind(&from) + .bind(&to) + .fetch_one(&self.data_pool) + .await + .map_err(|e| StorageError::Internal(e.to_string())) + } +} diff --git a/crates/storage-sqlite/src/stream.rs b/crates/storage-sqlite/src/stream.rs index bf2b8c2df..56e576b91 100644 --- a/crates/storage-sqlite/src/stream.rs +++ b/crates/storage-sqlite/src/stream.rs @@ -83,7 +83,9 @@ impl StreamEngine for SqliteEngine { // all. The sequence bound sits in the join condition so the LEFT // JOIN keeps its one row for an idle shard. Whether the shard // belongs to the stream the ARN's label names is decided in Rust - // from the table's label and the shard id (`shard_in_stream`). + // from the table's label and the shard id (`shard_in_stream`), + // with a second query for a legacy id only: whether the table has + // a labelled shard. let rows: Vec<(String, Option, String, Option)> = if let Some(after) = after { sqlx::query_as( @@ -127,7 +129,10 @@ impl StreamEngine for SqliteEngine { row_shard_id, table_label.as_deref(), &stream_label, - ) { + || self.table_has_labelled_shard(table_id), + ) + .await? + { return Err(extenddb_storage::util::stream_does_not_exist()); } @@ -328,3 +333,23 @@ impl StreamEngine for SqliteEngine { }) } } + +impl SqliteEngine { + /// Whether table `table_id` has a labelled shard row + /// (`stream_lineage::labelled_shard_id_range`), which makes its legacy + /// rows an earlier, closed stream (`util::shard_in_stream`). + async fn table_has_labelled_shard(&self, table_id: &str) -> Result { + let (from, to) = stream_lineage::labelled_shard_id_range(table_id); + let found: i64 = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM stream_shards \ + WHERE table_id = ? AND shard_id >= ? AND shard_id < ?)", + ) + .bind(table_id) + .bind(&from) + .bind(&to) + .fetch_one(&self.pool) + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + Ok(found != 0) + } +} diff --git a/crates/storage-sqlite/src/stream_tests.rs b/crates/storage-sqlite/src/stream_tests.rs index 8ba92a55c..ba95b7ae9 100644 --- a/crates/storage-sqlite/src/stream_tests.rs +++ b/crates/storage-sqlite/src/stream_tests.rs @@ -289,3 +289,176 @@ async fn a_malformed_stream_arn_is_a_validation_error() { .expect_err("malformed ARN"); assert!(matches!(err, StorageError::Validation(_)), "{err:?}"); } + +/// Shard `i` of the stream `label` on `table_id`, in the shape +/// `init_stream_shards` writes since streams were named in shard ids. +fn labelled_shard(table_id: &str, label: &str, i: u32) -> String { + extenddb_storage::stream_lineage::shard_id( + table_id, + label, + extenddb_core::types::StreamViewType::KeysOnly, + i, + ) + .expect("label shape") +} + +/// Four shard rows of the stream `label` on `table_id`, closed at +/// `ending_sequence_number` when it is given. +async fn labelled_shards(engine: &SqliteEngine, table_id: &str, label: &str, end: Option) { + for i in 0..4u32 { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number, \ + ending_sequence_number) VALUES (?, ?, ?, ?)", + ) + .bind(labelled_shard(table_id, label, i)) + .bind(table_id) + .bind(format!("{:021}", 0)) + .bind(end.map(|e| format!("{e:021}"))) + .execute(&engine.pool) + .await + .expect("shard"); + } +} + +/// A table whose stream was disabled and enabled again: the closed stream's +/// labelled shards, then the enabled stream's, with the table's label on the +/// new stream. Returns the table id. +async fn re_enabled_table(engine: &SqliteEngine, old: &str, new: &str) -> String { + let table_id = stream_table(engine, ACCOUNT, "t", new).await; + sqlx::query("DELETE FROM stream_shards WHERE table_id = ?") + .bind(&table_id) + .execute(&engine.pool) + .await + .expect("drop the legacy rows the fixture wrote"); + labelled_shards(engine, &table_id, old, Some(2)).await; + labelled_shards(engine, &table_id, new, None).await; + write_records( + engine, + &labelled_shard(&table_id, old, 0), + &table_id, + &[1, 2], + ) + .await; + write_records(engine, &labelled_shard(&table_id, new, 0), &table_id, &[3]).await; + table_id +} + +#[tokio::test] +async fn a_closed_stream_stays_readable_after_the_table_has_a_newer_one() { + let engine = engine().await; + let (old, new) = ("2026-01-01T00:00:00.000", "2026-02-02T00:00:00.000"); + let table_id = re_enabled_table(&engine, old, new).await; + let old_arn = stream_arn(REGION, ACCOUNT, "t", old); + let new_arn = stream_arn(REGION, ACCOUNT, "t", new); + + let (records, last) = engine + .get_stream_records( + ACCOUNT, + &old_arn, + &labelled_shard(&table_id, old, 0), + None, + 10, + ) + .await + .expect("the closed stream reads under its own ARN"); + assert_eq!(records.len(), 2); + assert_eq!(last.as_deref(), Some(format!("{:021}", 2).as_str())); + + let (records, _) = engine + .get_stream_records( + ACCOUNT, + &new_arn, + &labelled_shard(&table_id, new, 0), + None, + 10, + ) + .await + .expect("the new stream reads under its own ARN"); + assert_eq!(records.len(), 1); + assert_eq!(records[0].dynamodb.sequence_number, format!("{:021}", 3)); +} + +#[tokio::test] +async fn the_newer_arn_with_an_older_streams_shard_id_is_not_found() { + let engine = engine().await; + let (old, new) = ("2026-01-01T00:00:00.000", "2026-02-02T00:00:00.000"); + let table_id = re_enabled_table(&engine, old, new).await; + let old_arn = stream_arn(REGION, ACCOUNT, "t", old); + let new_arn = stream_arn(REGION, ACCOUNT, "t", new); + + // An iterator pairing the new ARN with a closed shard, and one pairing the + // closed stream's ARN with a new shard: neither shard is the stream's. + assert_stream_not_found( + engine + .get_stream_records( + ACCOUNT, + &new_arn, + &labelled_shard(&table_id, old, 0), + None, + 10, + ) + .await, + ); + assert_stream_not_found( + engine + .get_stream_records( + ACCOUNT, + &old_arn, + &labelled_shard(&table_id, new, 0), + None, + 10, + ) + .await, + ); +} + +#[tokio::test] +async fn the_newer_arn_with_a_legacy_shard_of_the_closed_stream_is_not_found() { + // A stream created before shard ids named their stream, disabled and + // enabled again under the labelled scheme: the legacy rows are closed, and + // the table's label is the new labelled stream's. + let engine = engine().await; + let new = "2026-02-02T00:00:00.000"; + let table_id = stream_table(&engine, ACCOUNT, "t", new).await; + write_records(&engine, &shard(&table_id, 0), &table_id, &[1, 2]).await; + sqlx::query( + "UPDATE stream_shards SET ending_sequence_number = ? WHERE table_id = ? AND shard_id = ?", + ) + .bind(format!("{:021}", 2)) + .bind(&table_id) + .bind(shard(&table_id, 0)) + .execute(&engine.pool) + .await + .expect("close shard 0"); + sqlx::query( + "UPDATE stream_shards SET ending_sequence_number = starting_sequence_number \ + WHERE table_id = ? AND ending_sequence_number IS NULL", + ) + .bind(&table_id) + .execute(&engine.pool) + .await + .expect("close the other legacy shards"); + labelled_shards(&engine, &table_id, new, None).await; + write_records(&engine, &labelled_shard(&table_id, new, 0), &table_id, &[3]).await; + let new_arn = stream_arn(REGION, ACCOUNT, "t", new); + + for i in 0..4u32 { + assert_stream_not_found( + engine + .get_stream_records(ACCOUNT, &new_arn, &shard(&table_id, i), None, 10) + .await, + ); + } + // The control: the new stream's own shard reads under the new ARN. + let (records, _) = engine + .get_stream_records( + ACCOUNT, + &new_arn, + &labelled_shard(&table_id, new, 0), + None, + 10, + ) + .await + .expect("the new stream reads under its own ARN"); + assert_eq!(records.len(), 1); +} diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index fe14438b1..6d53bb1a1 100755 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -619,8 +619,10 @@ pub trait StreamEngine: Send + Sync { /// Cassandra after reading the table), and return `StorageError::TableNotFound` /// with the message `Requested resource not found: Stream does not exist` when the /// stream does not exist for that account or the shard is not one of that stream's - /// shards. A shard of another table, another account, or an earlier stream label on - /// the same table therefore yields no records. + /// shards (`util::shard_in_stream`). A shard of another table, another account, or + /// another stream of the same table therefore yields no records; a closed stream's + /// shards keep reading under the closed stream's ARN after the table has a newer + /// stream. fn get_stream_records( &self, account_id: &str, diff --git a/crates/storage/src/stream_lineage.rs b/crates/storage/src/stream_lineage.rs index 3fb18e8e7..99676074c 100644 --- a/crates/storage/src/stream_lineage.rs +++ b/crates/storage/src/stream_lineage.rs @@ -267,6 +267,42 @@ pub fn parse_shard_id(table_id: &str, shard_id: &str) -> Option<(ShardStream, u3 Some((ShardStream::Legacy, suffix.parse().ok()?)) } +/// The byte-order range `[from, to)` that holds the labelled shard ids of +/// table `table_id` and none of its legacy ones, for the backends that write +/// 16-digit suffixes (SQLite and PostgreSQL). A labelled suffix starts with +/// the label's 13 milliseconds digits, not all zeros, so it sorts at or above +/// `0000000000001`; a legacy `{index:016}` suffix starts with 13 zeros and +/// sorts below. The upper end keeps the `shardId-{table_name}-{index:016}` +/// form out, which sorts anywhere relative to the table id. +#[must_use] +pub fn labelled_shard_id_range(table_id: &str) -> (String, String) { + ( + format!("shardId-{table_id}-0000000000001"), + format!("shardId-{table_id}."), + ) +} + +/// A regular expression matching the labelled shard ids of table `table_id` +/// and none of its legacy ones, for MongoDB, whose legacy ids have 12-digit +/// suffixes that no byte-order bound separates from a labelled one. Table ids +/// are UUIDs, so the id needs no escaping. +#[must_use] +pub fn labelled_shard_id_pattern(table_id: &str) -> String { + format!("^shardId-{table_id}-(?!0{{13}})[0-9]{{16}}$") +} + +/// Whether any of `shards` is a labelled shard of table `table_id`. +#[must_use] +pub fn has_labelled_shard(table_id: &str, shards: &[ShardRow]) -> bool { + shards.iter().any(|r| { + r.table_id == table_id + && matches!( + parse_shard_id(table_id, &r.shard_id), + Some((ShardStream::Labelled { .. }, _)) + ) + }) +} + /// One shard row as a backend stores it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ShardRow { diff --git a/crates/storage/src/stream_lineage_tests.rs b/crates/storage/src/stream_lineage_tests.rs index b4e4daa02..917e5c062 100644 --- a/crates/storage/src/stream_lineage_tests.rs +++ b/crates/storage/src/stream_lineage_tests.rs @@ -463,3 +463,41 @@ fn an_enable_re_read_sees_whether_its_stream_is_still_current() { assert!(!enable_still_current(Some(&table(Some(NEW), None)), NEW)); assert!(!enable_still_current(None, NEW)); } + +#[test] +fn the_labelled_range_holds_labelled_ids_and_no_legacy_ones() { + let (from, to) = labelled_shard_id_range(TID); + let inside = |id: &str| id >= from.as_str() && id < to.as_str(); + for view in [StreamViewType::KeysOnly, StreamViewType::NewAndOldImages] { + for i in 0..SHARDS_PER_STREAM { + assert!(inside(&shard_id(TID, OLD, view, i).unwrap())); + } + } + for i in 0..SHARDS_PER_STREAM { + assert!(!inside(&format!("shardId-{TID}-{i:016}"))); + } + // The name forms sort on either side of the table id. + assert!(!inside("shardId-Orders-0000000000000002")); + assert!(!inside("shardId-00-orders-0000000000000002")); + assert!(!inside("shardId-zz-orders-0000000000000002")); +} + +#[test] +fn a_table_has_a_labelled_shard_once_an_enable_wrote_one() { + let legacy: Vec = (0..SHARDS_PER_STREAM) + .map(|i| row(format!("shardId-{TID}-{i:012}"), Some("1"))) + .collect(); + assert!(!has_labelled_shard(TID, &legacy)); + let mut both = legacy; + both.extend(stream_rows(NEW, StreamViewType::KeysOnly, None)); + assert!(has_labelled_shard(TID, &both)); + // Another table's labelled rows do not count. + let foreign = vec![ShardRow { + table_id: "other".to_owned(), + ..row( + shard_id("other", OLD, StreamViewType::KeysOnly, 0).unwrap(), + None, + ) + }]; + assert!(!has_labelled_shard(TID, &foreign)); +} diff --git a/crates/storage/src/util/shard_stream.rs b/crates/storage/src/util/shard_stream.rs index 93982c78b..113198737 100644 --- a/crates/storage/src/util/shard_stream.rs +++ b/crates/storage/src/util/shard_stream.rs @@ -10,54 +10,198 @@ //! label names. The rule lives in one place so a change to how streams are //! identified touches one function and its test. +use std::future::Future; + +use crate::stream_lineage::{ShardStream, parse_shard_id}; + /// Whether shard `shard_id`, a row of table `table_id` whose current stream /// label is `table_label` (`None` when the table never had a stream), belongs /// to the stream labelled `arn_label`. /// -/// Today a table has one stream and every shard row of the table belongs to -/// it, so the shard is the stream's exactly when the ARN's label is the -/// table's current label. `table_id` and `shard_id` are part of the contract -/// for a scheme that names the stream inside the shard id; this rule does not -/// read them. -#[must_use] -pub fn shard_in_stream( +/// A table has a sequence of streams, one per enable, and each stream's +/// identity is in its shard ids ([`crate::stream_lineage::shard_id`]). A +/// labelled id belongs to the label it carries, so a closed stream's shards +/// stay readable under the closed stream's ARN after the table has a newer +/// stream, and an iterator pairing the newer ARN with one of them is refused. +/// An id from before streams were named in shard ids (the +/// `shardId-{table_id}-{index}` and `shardId-{table_name}-{index}` shapes) +/// belongs to the table's current stream while the table has no labelled +/// shard: it is the ARN's when the ARN's label is the table's current label +/// and `table_has_labelled_shard` answers `false`. From a table's first enable +/// under the labelled scheme every later stream is labelled, so legacy rows +/// beside a labelled row are an earlier, closed stream, which the newer ARN +/// does not name. The row is already this table's (the callers key the shard +/// read on the table id), so any id that does not parse is a legacy one. +/// +/// `table_has_labelled_shard` reads the table's shard rows; it runs only for a +/// legacy id whose table label matches the ARN's, so a labelled id is decided +/// from the id alone. +/// +/// # Errors +/// +/// Whatever `table_has_labelled_shard` returns. +pub async fn shard_in_stream( table_id: &str, shard_id: &str, table_label: Option<&str>, arn_label: &str, -) -> bool { - let _ = (table_id, shard_id); - table_label == Some(arn_label) + table_has_labelled_shard: impl FnOnce() -> F, +) -> Result +where + F: Future>, +{ + match parse_shard_id(table_id, shard_id) { + Some((ShardStream::Labelled { label, .. }, _)) => Ok(label == arn_label), + Some((ShardStream::Legacy, _)) | None => { + if table_label != Some(arn_label) { + return Ok(false); + } + Ok(!table_has_labelled_shard().await?) + } + } } #[cfg(test)] mod tests { + use std::cell::Cell; + use std::convert::Infallible; + + use extenddb_core::types::StreamViewType; + use super::shard_in_stream; + use crate::stream_lineage::shard_id; const TABLE: &str = "0f3c1c1e-7b1d-4e3b-9a6e-4a2d3e5f6a7b"; const LABEL: &str = "2026-01-01T00:00:00.000"; const LATER: &str = "2026-02-02T00:00:00.000"; - fn shard(i: u32) -> String { + fn legacy_shard(i: u32) -> String { format!("shardId-{TABLE}-{i:016}") } + fn labelled_shard(label: &str, i: u32) -> String { + shard_id(TABLE, label, StreamViewType::KeysOnly, i).expect("label shape") + } + + /// `shard_in_stream` on a table whose shard rows include a labelled one + /// when `has_labelled` is set. + fn in_stream( + shard: &str, + table_label: Option<&str>, + arn_label: &str, + has_labelled: bool, + ) -> bool { + let answer = futures::executor::block_on(shard_in_stream( + TABLE, + shard, + table_label, + arn_label, + || async move { Ok::(has_labelled) }, + )); + match answer { + Ok(v) => v, + Err(never) => match never {}, + } + } + + #[test] + fn a_labelled_shard_belongs_to_the_stream_its_id_names() { + assert!(in_stream( + &labelled_shard(LABEL, 0), + Some(LABEL), + LABEL, + true + )); + assert!(in_stream( + &labelled_shard(LABEL, 3), + Some(LABEL), + LABEL, + true + )); + } + + #[test] + fn a_closed_stream_stays_readable_after_the_table_has_a_newer_one() { + // The table was disabled and enabled again: its label moved on to the + // new stream, and the closed stream's shards still answer to the + // closed stream's ARN. + assert!(in_stream( + &labelled_shard(LABEL, 0), + Some(LATER), + LABEL, + true + )); + assert!(in_stream( + &labelled_shard(LATER, 0), + Some(LATER), + LATER, + true + )); + } + + #[test] + fn the_newer_arn_with_an_older_streams_shard_id_is_not_its_stream() { + // An iterator forged from the new ARN and an old shard id, or the + // other way round, names a shard the stream does not have. + assert!(!in_stream( + &labelled_shard(LABEL, 0), + Some(LATER), + LATER, + true + )); + assert!(!in_stream( + &labelled_shard(LATER, 0), + Some(LATER), + LABEL, + true + )); + } + + #[test] + fn a_legacy_shard_belongs_to_its_tables_current_stream() { + assert!(in_stream(&legacy_shard(0), Some(LABEL), LABEL, false)); + assert!(in_stream(&legacy_shard(3), Some(LABEL), LABEL, false)); + // The shape SQLite and PostgreSQL wrote before v0.1.14 names the table, + // not its id; the row is the table's, so it is a legacy shard too. + assert!(in_stream( + "shardId-Orders-0000000000000002", + Some(LABEL), + LABEL, + false + )); + } + #[test] - fn a_shard_belongs_to_its_tables_current_stream() { - assert!(shard_in_stream(TABLE, &shard(0), Some(LABEL), LABEL)); - assert!(shard_in_stream(TABLE, &shard(3), Some(LABEL), LABEL)); + fn an_earlier_label_on_legacy_shards_is_not_the_current_stream() { + // The table took a new label while keeping its legacy shard rows: the + // old ARN no longer names a stream of the table. + assert!(!in_stream(&legacy_shard(0), Some(LATER), LABEL, false)); + // The new label is a labelled stream of its own (an enable gave it + // shards), so the legacy rows are the earlier, closed stream and the + // newer ARN does not name them. + assert!(!in_stream(&legacy_shard(0), Some(LATER), LATER, true)); } #[test] - fn an_earlier_label_on_the_same_shards_is_not_the_current_stream() { - // The table took a new label while keeping its shard rows: the old - // ARN no longer names a stream of the table. - assert!(!shard_in_stream(TABLE, &shard(0), Some(LATER), LABEL)); - assert!(shard_in_stream(TABLE, &shard(0), Some(LATER), LATER)); + fn a_labelled_shard_is_decided_without_reading_the_tables_shards() { + let reads = Cell::new(0); + let answer = futures::executor::block_on(shard_in_stream( + TABLE, + &labelled_shard(LABEL, 0), + Some(LABEL), + LABEL, + || { + reads.set(reads.get() + 1); + async { Ok::(true) } + }, + )); + assert!(matches!(answer, Ok(true))); + assert_eq!(reads.get(), 0); } #[test] fn a_table_without_a_stream_owns_no_stream_shard() { - assert!(!shard_in_stream(TABLE, &shard(0), None, LABEL)); + assert!(!in_stream(&legacy_shard(0), None, LABEL, false)); + assert!(!in_stream(&labelled_shard(LABEL, 0), None, LATER, false)); } } From 690073dc58d5e868b46ef0ec9e298cf0df79fe27 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:34:46 +0000 Subject: [PATCH 03/15] test(streams): read re-enabled and relabelled streams by the shard ids they carry The wire test that pinned a re-enable to the same stream ARN moves to the new ARN the service answers: the iterators issued before the disable read the closed stream (the same records a fresh iterator on each of its shards reads) and none of the keys written after the enable, which are in the new stream's shards alone. The Cassandra direct tests read a stream's records by the stream's own shard id for the partition a key lands in, not by the partition id, since get_stream_records resolves the shard row the id names. The ownership tests' relabel moves the table's shard rows to the ids the new label gives them, as a shard id carries its stream's label, and the disable test checks that a token pairing one stream's ARN with the other stream's shard id is refused. --- .../tests/direct/stream_disable.rs | 16 +++ .../tests/direct/stream_ownership.rs | 109 +++++++++++++++--- .../storage-cassandra/tests/direct/streams.rs | 30 ++++- tests/python/test_stream_shard_ownership.py | 23 +++- 4 files changed, 153 insertions(+), 25 deletions(-) diff --git a/crates/storage-cassandra/tests/direct/stream_disable.rs b/crates/storage-cassandra/tests/direct/stream_disable.rs index 6dd0a5406..daa87ba4a 100644 --- a/crates/storage-cassandra/tests/direct/stream_disable.rs +++ b/crates/storage-cassandra/tests/direct/stream_disable.rs @@ -250,6 +250,22 @@ async fn disable_closes_every_shard_and_re_enable_creates_a_new_stream() { .len(); } assert_eq!(on_new, 1); + // A token pairing one stream's ARN with the other stream's shard id names + // a shard the stream does not have. + for (arn, shard) in [ + (&new_arn, &closed.shards[0].shard_id), + (&old_arn, &new.shards[0].shard_id), + ] { + match engine + .get_stream_records(&account, arn, shard, None, 1000) + .await + { + Err(StorageError::TableNotFound(m)) => { + assert_eq!(m, extenddb_storage::util::STREAM_DOES_NOT_EXIST); + } + other => panic!("{other:?}"), + } + } // LATEST on a closed shard resolves to its last record, not to the new // stream's. for shard in &closed.shards { diff --git a/crates/storage-cassandra/tests/direct/stream_ownership.rs b/crates/storage-cassandra/tests/direct/stream_ownership.rs index fa1170c59..00af0aa4c 100644 --- a/crates/storage-cassandra/tests/direct/stream_ownership.rs +++ b/crates/storage-cassandra/tests/direct/stream_ownership.rs @@ -5,36 +5,91 @@ //! //! The catalog read is keyed on the caller's account and the table name the //! ARN carries, the shard read on that table's id, and the stream label is -//! decided in Rust from the table's current label. Each test here removes one -//! of those from the picture (same label on two tables, same name in two -//! accounts, a new label on the same shards) so that one predicate alone +//! decided in Rust from the label the shard id carries. Each test here removes +//! one of those from the picture (same label on two tables, same name in two +//! accounts, a new label on the same partitions) so that one predicate alone //! decides the answer. use std::collections::BTreeMap; use std::sync::Arc; +use cdrs_tokio::types::IntoRustByName; use extenddb_core::types::{AttributeValue, StreamViewType, TableKeyInfo}; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ShardStream, parse_shard_id, shard_id}; use extenddb_storage::util::stream_arn; use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, StreamRecordsResult}; use extenddb_storage_cassandra::CassandraEngine; use crate::helpers::setup_engine; -use crate::streams::{setup_stream_table, shard_for}; +use crate::streams::{setup_stream_table, stream_shard_for}; const REGION: &str = "us-east-1"; const LABEL: &str = "2026-01-01T00:00:00.000"; const LATER_LABEL: &str = "2026-02-02T00:00:00.000"; -/// Pin a table's stream label so two tables can share one. +/// Pin a table's stream label so two tables can share one. A shard id carries +/// its stream's label, so each shard row of the table moves to the id the +/// stream `label` gives its index, with the row's sequence range kept. async fn relabel(engine: &CassandraEngine, key_info: &TableKeyInfo, label: &str) { + let session = engine.session_arc(); + let account_ks = engine.account_keyspace(&key_info.account_id); + let rows = session + .query_with_values( + &format!( + "SELECT shard_id, starting_sequence_number, ending_sequence_number \ + FROM {account_ks}.stream_shards WHERE table_id = ?" + ), + cdrs_tokio::query_values!(key_info.table_id.as_str()), + ) + .await + .expect("shard rows") + .response_body() + .expect("body") + .into_rows() + .unwrap_or_default(); + for row in rows { + let old: String = row.get_r_by_name("shard_id").expect("shard_id"); + let start: String = row + .get_r_by_name("starting_sequence_number") + .expect("starting_sequence_number"); + let end: Option = row.get_by_name("ending_sequence_number").ok().flatten(); + let Some((ShardStream::Labelled { view_type, .. }, index)) = + parse_shard_id(&key_info.table_id, &old) + else { + panic!("{old} is not a labelled shard of {}", key_info.table_id); + }; + let new = shard_id(&key_info.table_id, label, view_type, index).expect("label shape"); + session + .query_with_values( + &format!("DELETE FROM {account_ks}.stream_shards WHERE shard_id = ?"), + cdrs_tokio::query_values!(old.as_str()), + ) + .await + .expect("delete the old shard row"); + session + .query_with_values( + &format!( + "INSERT INTO {account_ks}.stream_shards \ + (shard_id, table_id, starting_sequence_number, ending_sequence_number) \ + VALUES (?, ?, ?, ?)" + ), + cdrs_tokio::query_values!( + new.as_str(), + key_info.table_id.as_str(), + start.as_str(), + end + ), + ) + .await + .expect("insert the relabelled shard row"); + } let catalog_keyspace = engine.catalog_keyspace(); let query = format!( "UPDATE {catalog_keyspace}.tables SET stream_label = ? \ WHERE account_id = ? AND table_name = ?" ); - engine - .session_arc() + session .query_with_values( &query, cdrs_tokio::query_values!( @@ -97,7 +152,8 @@ async fn stream_table_in( } } -/// One item written under `pk`, which lands in `shard_for(pk, table_id)`. +/// One item written under `pk`, which lands in the partition `shard_for(pk, +/// table_id)` names; the stream's shard for it is `shard_of(pk, table, label)`. async fn put(engine: &CassandraEngine, key_info: &TableKeyInfo, pk: &str, val: u32) { let mut item = BTreeMap::new(); item.insert("id".to_string(), AttributeValue::S(pk.to_string())); @@ -124,6 +180,17 @@ fn arn_of(key_info: &TableKeyInfo, label: &str) -> String { stream_arn(REGION, &key_info.account_id, &key_info.table_name, label) } +/// The shard of the stream `label` of `key_info`'s table that a record written +/// under `pk` lands in. +fn shard_of(pk: &str, key_info: &TableKeyInfo, label: &str) -> String { + let view_type = key_info + .stream_specification + .as_ref() + .and_then(|s| s.stream_view_type) + .expect("a stream-enabled table"); + stream_shard_for(pk, &key_info.table_id, label, view_type) +} + fn assert_stream_not_found(result: StreamRecordsResult) { match result { Err(StorageError::TableNotFound(msg)) => { @@ -147,8 +214,8 @@ async fn test_another_tables_shard_with_the_same_label_is_not_found() { relabel(&engine, &b, LABEL).await; put(&engine, &a, "pk-a", 1).await; put(&engine, &b, "pk-b", 1).await; - let shard_a = shard_for("pk-a", &a.table_id); - let shard_b = shard_for("pk-b", &b.table_id); + let shard_a = shard_of("pk-a", &a, LABEL); + let shard_b = shard_of("pk-b", &b, LABEL); let arn_a = arn_of(&a, LABEL); let arn_b = arn_of(&b, LABEL); @@ -169,7 +236,7 @@ async fn test_another_tables_shard_with_the_same_label_is_not_found() { .get_stream_records( &a.account_id, &arn_a, - "shardId-forged-000000000000", + "shardId-forged-0000000000000000", None, 10, ) @@ -201,8 +268,8 @@ async fn test_another_accounts_same_named_stream_is_not_found() { relabel(&engine, &other, LABEL).await; put(&engine, &own, "pk-o", 1).await; put(&engine, &other, "pk-o", 1).await; - let own_shard = shard_for("pk-o", &own.table_id); - let other_shard = shard_for("pk-o", &other.table_id); + let own_shard = shard_of("pk-o", &own, LABEL); + let other_shard = shard_of("pk-o", &other, LABEL); // From both callers, so whichever table a lookup without the account // would pick, one caller reads the other account's shard. @@ -228,8 +295,9 @@ async fn test_another_accounts_same_named_stream_is_not_found() { } } -/// The table takes a new stream label while keeping its shard rows: the old -/// ARN's iterator is refused and the new ARN reads every record. +/// The table's stream takes a new label while keeping its partitions: the old +/// ARN's iterator, and the old shard id under the new ARN, are refused, and +/// the new stream's shard reads every record of its partition. #[tokio::test] async fn test_an_earlier_label_on_the_same_shards_is_not_found() { if crate::helpers::skip_without_cassandra() { @@ -239,7 +307,7 @@ async fn test_an_earlier_label_on_the_same_shards_is_not_found() { let (t, _) = setup_stream_table(&engine, "Relabel").await; relabel(&engine, &t, LABEL).await; put(&engine, &t, "pk-r", 1).await; - let shard = shard_for("pk-r", &t.table_id); + let shard = shard_of("pk-r", &t, LABEL); let old_arn = arn_of(&t, LABEL); engine .get_stream_records(&t.account_id, &old_arn, &shard, None, 10) @@ -248,13 +316,20 @@ async fn test_an_earlier_label_on_the_same_shards_is_not_found() { relabel(&engine, &t, LATER_LABEL).await; put(&engine, &t, "pk-r", 2).await; + let new_arn = arn_of(&t, LATER_LABEL); + let new_shard = shard_of("pk-r", &t, LATER_LABEL); assert_stream_not_found( engine .get_stream_records(&t.account_id, &old_arn, &shard, None, 10) .await, ); + assert_stream_not_found( + engine + .get_stream_records(&t.account_id, &new_arn, &shard, None, 10) + .await, + ); let (records, _) = engine - .get_stream_records(&t.account_id, &arn_of(&t, LATER_LABEL), &shard, None, 10) + .get_stream_records(&t.account_id, &new_arn, &new_shard, None, 10) .await .expect("new label reads"); assert_eq!(records.len(), 2); diff --git a/crates/storage-cassandra/tests/direct/streams.rs b/crates/storage-cassandra/tests/direct/streams.rs index 7fcdc04a5..c7e65dc70 100644 --- a/crates/storage-cassandra/tests/direct/streams.rs +++ b/crates/storage-cassandra/tests/direct/streams.rs @@ -143,6 +143,22 @@ pub(crate) fn shard_for(pk: &str, table_id: &str) -> String { extenddb_storage_cassandra::stream_util::assign_shard_id(pk, table_id) } +/// The shard of the stream `label` (view type `view_type`) of `table_id` that +/// records written under `pk` land in: the stream's shard for the partition +/// `shard_for` names, which is the id `get_stream_records` reads it by. +pub(crate) fn stream_shard_for( + pk: &str, + table_id: &str, + label: &str, + view_type: StreamViewType, +) -> String { + let (_, index) = + extenddb_storage::stream_lineage::parse_shard_id(table_id, &shard_for(pk, table_id)) + .expect("the partition id is the table's"); + extenddb_storage::stream_lineage::shard_id(table_id, label, view_type, index) + .expect("label shape") +} + #[tokio::test] async fn test_put_item_insert_writes_stream_record() { if crate::helpers::skip_without_cassandra() { @@ -525,7 +541,12 @@ async fn test_get_stream_records_returns_written_records() { .unwrap(); } - let shard_id = shard_for("pk-read", &key_info.table_id); + let shard_id = stream_shard_for( + "pk-read", + &key_info.table_id, + &stream_label, + StreamViewType::NewAndOldImages, + ); let (records, last_seq) = engine .get_stream_records(&key_info.account_id, &arn, &shard_id, None, 10) .await @@ -572,7 +593,12 @@ async fn test_get_stream_records_after_sequence_paginates() { .unwrap(); } - let shard_id = shard_for("pk-page", &key_info.table_id); + let shard_id = stream_shard_for( + "pk-page", + &key_info.table_id, + &stream_label, + StreamViewType::NewAndOldImages, + ); let (first_page, last_seq) = engine .get_stream_records(&key_info.account_id, &arn, &shard_id, None, 2) .await diff --git a/tests/python/test_stream_shard_ownership.py b/tests/python/test_stream_shard_ownership.py index 6a9974b81..3ebfbde79 100644 --- a/tests/python/test_stream_shard_ownership.py +++ b/tests/python/test_stream_shard_ownership.py @@ -143,12 +143,15 @@ def test_allowed_stream_arn_with_another_streams_shard_reads_nothing( for name in (name_a, name_b): wait_for_deleted(dynamodb_client, name) - def test_iterator_reads_only_while_its_arn_is_the_tables_stream( + def test_iterator_reads_only_the_stream_its_arn_names( self, dynamodb_client, streams_client ): """An iterator issued before the stream was disabled and enabled again - keeps reading while the table keeps its stream ARN: across the four - shards it reads the same records a fresh iterator on each shard reads.""" + keeps reading the stream it was issued for. Re-enabling creates a + stream with a new ARN, as on the service, so the writes after it are + in the new stream's shards alone: across the four old shards the old + iterators read the same records a fresh iterator on each reads, and + none of the new keys.""" name, old_arn = _create_stream_table(dynamodb_client) try: old_shards = _shards(streams_client, old_arn) @@ -160,14 +163,14 @@ def test_iterator_reads_only_while_its_arn_is_the_tables_stream( dynamodb_client.update_table(TableName=name, StreamSpecification=STREAM_SPEC) wait_for_active(dynamodb_client, name) new_arn = dynamodb_client.describe_table(TableName=name)["Table"]["LatestStreamArn"] - # A re-enabled stream with a new ARN, as on the service, becomes reachable with #515. - assert new_arn == old_arn, f"re-enabling made a new stream ARN: {new_arn}" + assert new_arn != old_arn, f"re-enabling kept the stream ARN: {new_arn}" new_keys = {f"n{i}" for i in range(8)} for key in sorted(new_keys): dynamodb_client.put_item(TableName=name, Item={"pk": {"S": key}}) time.sleep(1) new_shards = _shards(streams_client, new_arn) + assert not set(new_shards) & set(old_shards), (old_shards, new_shards) new_records = [ r for s in new_shards @@ -183,7 +186,15 @@ def test_iterator_reads_only_while_its_arn_is_the_tables_stream( old_seqs = sorted( r["dynamodb"]["SequenceNumber"] for p in old_pages.values() for r in p["Records"] ) - assert old_seqs == sorted(r["dynamodb"]["SequenceNumber"] for r in new_records) + fresh_seqs = sorted( + r["dynamodb"]["SequenceNumber"] + for s in old_shards + for r in streams_client.get_records( + ShardIterator=_trim_horizon(streams_client, old_arn, s) + )["Records"] + ) + assert old_seqs == fresh_seqs + assert not set(old_seqs) & {r["dynamodb"]["SequenceNumber"] for r in new_records} finally: dynamodb_client.delete_table(TableName=name) wait_for_deleted(dynamodb_client, name) From ff46ea465226024aa28f0d04722bcc4ba47f6a67 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:38:13 +0000 Subject: [PATCH 04/15] fix(engine): a StreamSpecification that is not an object stays a SerializationException The StreamEnabled member check ran on any non-null StreamSpecification, so a string, number or array there answered the null-member ValidationException where main answered SerializationException from deserialisation. The check now applies to objects only. The doc block that had landed inside deserialize_error's comment documents the member check it describes. --- crates/engine/src/lib.rs | 24 +++++++++------- crates/engine/src/update_table_tests.rs | 38 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 18e87f1ec..cf8712c3f 100755 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -151,18 +151,15 @@ pub(crate) fn sanitize_storage_error(e: extenddb_storage::error::StorageError) - DynamoDbError::InternalServerError("Internal server error".to_owned()) } -/// Map a serde deserialization error to the appropriate `DynamoDB` error type. -/// -/// Enum validation errors (produced by custom Deserialize impls) contain -/// "validation error detected" and should be returned as `ValidationException`. -/// All other deserialization failures are `SerializationException`. -/// A `StreamSpecification` member without `StreamEnabled`, on `CreateTable` -/// and `UpdateTable` alike (measured 2026-10-24). Checked before the body is -/// deserialised, where the missing field would otherwise surface as a -/// serialization error; its order against the other request checks is not -/// measured. +/// The null-member sentence for a `StreamSpecification` object without +/// `StreamEnabled`, on `CreateTable` and `UpdateTable` alike (measured +/// 2026-10-24). Checked before the body is deserialised, where the missing +/// field would otherwise surface as a serialization error; its order against +/// the other request checks is not measured. A `StreamSpecification` that is +/// not an object is left to deserialisation, which refuses it as a +/// `SerializationException`. pub(crate) fn require_stream_enabled_member(body: &serde_json::Value) -> Result<(), DynamoDbError> { - if let Some(spec) = body.get("StreamSpecification").filter(|v| !v.is_null()) + if let Some(spec) = body.get("StreamSpecification").filter(|v| v.is_object()) && spec .get("StreamEnabled") .is_none_or(serde_json::Value::is_null) @@ -174,6 +171,11 @@ pub(crate) fn require_stream_enabled_member(body: &serde_json::Value) -> Result< Ok(()) } +/// Map a serde deserialization error to the appropriate `DynamoDB` error type. +/// +/// Enum validation errors (produced by custom Deserialize impls) contain +/// "validation error detected" and should be returned as `ValidationException`. +/// All other deserialization failures are `SerializationException`. pub(crate) fn deserialize_error(e: serde_json::Error) -> DynamoDbError { let msg = e.to_string(); if msg.contains("validation error detected") diff --git a/crates/engine/src/update_table_tests.rs b/crates/engine/src/update_table_tests.rs index 642a95a26..9edcfd80a 100644 --- a/crates/engine/src/update_table_tests.rs +++ b/crates/engine/src/update_table_tests.rs @@ -213,3 +213,41 @@ fn an_internal_failure_is_still_an_internal_server_error() { let err = update_table_err_to_dynamo(StorageError::Internal("disk on fire".to_owned()), "t"); assert!(matches!(err, DynamoDbError::InternalServerError(_))); } + +/// Measured 2026-10-24: a `StreamSpecification` object without `StreamEnabled` +/// gets the null-member sentence. A member that is not an object (a string, a +/// number, an array) is not that case: it is left to deserialisation, which +/// refuses it as a `SerializationException`, as before the member check. +#[test] +fn a_stream_specification_that_is_not_an_object_is_a_serialization_error() { + let null_member = "1 validation error detected: Value null at 'streamSpecification.streamEnabled' failed to satisfy constraint: Member must not be null"; + for spec in [ + serde_json::json!({}), + serde_json::json!({"StreamViewType": "KEYS_ONLY"}), + serde_json::json!({"StreamEnabled": null}), + ] { + let body = serde_json::json!({"TableName": "t", "StreamSpecification": spec}); + match crate::require_stream_enabled_member(&body) { + Err(DynamoDbError::ValidationException(m)) => assert_eq!(m, null_member), + other => panic!("{other:?}"), + } + } + for spec in [ + serde_json::json!("x"), + serde_json::json!(1), + serde_json::json!([]), + serde_json::json!(true), + ] { + let body = serde_json::json!({"TableName": "t", "StreamSpecification": spec}); + crate::require_stream_enabled_member(&body).expect("not the member check's case"); + let err = serde_json::from_value::(body) + .map_err(crate::deserialize_error) + .expect_err("a non-object specification does not deserialise"); + assert!( + matches!(err, DynamoDbError::SerializationException(_)), + "{err:?}" + ); + } + let body = serde_json::json!({"TableName": "t", "StreamSpecification": null}); + crate::require_stream_enabled_member(&body).expect("an absent member"); +} From cea6f303330fbeb496a6e2e625eeb3d712ca158f Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 10:41:55 +0000 Subject: [PATCH 05/15] fix(streams): apply the ShardId maximum only when the shard is not found GetShardIterator refused a ShardId over 65 characters before looking it up. The shards a release before v0.1.14 wrote are named after the table (shardId-{table_name}-{index:016}), so a table whose name is over 40 characters had its stream refused with a ValidationException after the upgrade, where the previous release read it. botocore enforces only the minimum, so SDK clients reach this check. The minimum and the SequenceNumber length are still checked first; the maximum is applied when the lookup finds no such shard, which is the answer the service gives since it never has a shard that long. A found shard over the maximum gets its iterator. --- crates/engine/src/streams.rs | 55 ++++++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/crates/engine/src/streams.rs b/crates/engine/src/streams.rs index 34305a850..6b457cbbd 100755 --- a/crates/engine/src/streams.rs +++ b/crates/engine/src/streams.rs @@ -114,19 +114,16 @@ pub async fn handle_get_shard_iterator( // The service's ShardId bound is 28 to 65 characters, checked before any // lookup with the model's constraint sentences (both measured 2026-10-24). // A SequenceNumber, when given, is at least 21 characters (measured with - // "0" on AT_SEQUENCE_NUMBER). + // "0" on AT_SEQUENCE_NUMBER). The maximum is applied once the shard is + // not found (`shard_lookup_error`): the shards a release before v0.1.14 + // wrote are named after the table, and a table name over 40 characters + // puts them past 65. let shard_len = input.shard_id.chars().count(); if shard_len < SHARD_ID_MIN_LEN { return Err(DynamoDbError::ValidationException( SHARD_ID_TOO_SHORT.to_owned(), )); } - if shard_len > SHARD_ID_MAX_LEN { - return Err(DynamoDbError::ValidationException(constraint( - "shardId", - "have length less than or equal to 65", - ))); - } if input .sequence_number .as_ref() @@ -142,7 +139,7 @@ pub async fn handle_get_shard_iterator( ctx.storage .validate_shard(&ctx.account_id, &input.stream_arn, &input.shard_id) .await - .map_err(storage_to_dynamo)?; + .map_err(|e| shard_lookup_error(shard_len, e))?; let seq = match input.shard_iterator_type { ShardIteratorType::TrimHorizon => String::new(), @@ -216,6 +213,22 @@ pub async fn handle_get_shard_iterator( serialize_output(&output) } +/// The answer when the shard a `GetShardIterator` names is not found: for a +/// `ShardId` of `shard_len` characters over the service's maximum, the model's +/// constraint sentence (the service checks it before any lookup, and never has +/// a shard that long); otherwise the storage layer's not-found answer. A shard +/// that long and found passes, so a stream from before v0.1.14 on a table whose +/// name is over 40 characters stays readable. +fn shard_lookup_error(shard_len: usize, err: StorageError) -> DynamoDbError { + if shard_len > SHARD_ID_MAX_LEN { + return DynamoDbError::ValidationException(constraint( + "shardId", + "have length less than or equal to 65", + )); + } + storage_to_dynamo(err) +} + /// Shard iterator expiration: 15 minutes (900 seconds), matching real `DynamoDB`. const SHARD_ITERATOR_EXPIRY_SECS: u64 = 900; @@ -586,6 +599,32 @@ mod tests { base64::Engine::encode(&base64::engine::general_purpose::STANDARD, s) } + #[test] + fn the_shard_id_maximum_applies_once_the_shard_is_not_found() { + // A shard of a table named with 41 characters, as a release before + // v0.1.14 wrote it: 8 + 41 + 1 + 16 = 66 characters. Found, it is not + // this function's case at all; not found, the model's sentence. + let long = format!("shardId-{}-{:016}", "t".repeat(41), 3); + assert_eq!(long.chars().count(), 66); + let sentence = "1 validation error detected: Value at 'shardId' failed to satisfy constraint: Member must have length less than or equal to 65"; + for err in [ + extenddb_storage::util::shard_does_not_exist(), + extenddb_storage::util::stream_does_not_exist(), + ] { + match super::shard_lookup_error(long.chars().count(), err) { + super::DynamoDbError::ValidationException(msg) => assert_eq!(msg, sentence), + other => panic!("expected ValidationException, got {other:?}"), + } + } + // Within the bound, the lookup's own answer stands. + match super::shard_lookup_error(65, extenddb_storage::util::shard_does_not_exist()) { + super::DynamoDbError::ResourceNotFoundException(msg) => { + assert_eq!(msg, extenddb_storage::util::SHARD_DOES_NOT_EXIST); + } + other => panic!("expected ResourceNotFoundException, got {other:?}"), + } + } + #[test] fn the_constraint_sentences_match_the_measured_ones() { assert_eq!( From 77232492df09573ff96f8c2149dbdb0a2e04f20c Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:12:37 +0000 Subject: [PATCH 06/15] fix(streams): a refused disable of a disabled stream closes the shards left open A release that stored {StreamEnabled: false} without closing the shards, or a close that failed after its catalog write, leaves a stream that DescribeStream reports as DISABLED with four open shards, and writes keep recording into it until the key info cache expires. A disable on that table was refused with "Table has no stream to disable" and changed nothing, so only an enable closed the shards. The decide step on each backend now closes the current label's open shards when the stored specification is already disabled, then answers the same refusal. SQLite closes in the request's transaction and commits it before answering; PostgreSQL, MongoDB and Cassandra close in the shard store, as the first disable would have, with no catalog write. Tests on SQLite and PostgreSQL seed that state: the refusal stands, the ending numbers are the last record or the starting number, a write after it records nothing, a second refusal has nothing left to close, and on SQLite an enable met with the same state closes the open shards too. --- .../storage-cassandra/src/stream_lifecycle.rs | 40 +++++- crates/storage-cassandra/src/update_table.rs | 11 +- .../storage-mongodb/src/stream_lifecycle.rs | 33 ++++- crates/storage-mongodb/src/table_engine.rs | 15 ++- .../storage-postgres/src/stream_lifecycle.rs | 24 +++- crates/storage-postgres/src/update_table.rs | 1 + .../storage-postgres/tests/stream_disable.rs | 110 +++++++++++++++ crates/storage-sqlite/src/stream_lifecycle.rs | 46 +++++-- .../src/stream_lifecycle_tests.rs | 125 ++++++++++++++++++ crates/storage-sqlite/src/update_table.rs | 14 +- crates/storage/src/stream_lineage.rs | 24 ++++ crates/storage/src/stream_lineage_tests.rs | 38 ++++++ 12 files changed, 448 insertions(+), 33 deletions(-) diff --git a/crates/storage-cassandra/src/stream_lifecycle.rs b/crates/storage-cassandra/src/stream_lifecycle.rs index aedd4816e..7b3a68344 100644 --- a/crates/storage-cassandra/src/stream_lifecycle.rs +++ b/crates/storage-cassandra/src/stream_lifecycle.rs @@ -21,9 +21,9 @@ use cdrs_tokio::types::IntoRustByName; use extenddb_core::types::{StreamSpecification, StreamViewType}; use extenddb_storage::error::StorageError; use extenddb_storage::stream_lineage::{ - SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, enable_still_current, - next_stream_label, open_shard_ids_of, parse_shard_id, shard_id, stored_specification, - stream_change, + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, + enable_still_current, next_stream_label, open_shard_ids_of, parse_shard_id, shard_id, + stored_specification, stream_change, }; use crate::CassandraEngine; @@ -187,6 +187,40 @@ impl PendingStream { } impl CassandraEngine { + /// [`PendingStream::decide`], and for a disable refused because the stored + /// specification is already disabled, the close of the current label's + /// open shards first (left by a release that did not close them, or by a + /// close that failed after the catalog write); the refusal is still the + /// answer. + pub(crate) async fn decide_stream_change_or_close( + &self, + row: &cdrs_tokio::types::rows::Row, + account_ks: &str, + table_id: &str, + table_name: &str, + requested: &StreamSpecification, + ) -> Result<(StreamColumns, PendingStream), StorageError> { + match PendingStream::decide(row, table_name, requested) { + Ok(decided) => Ok(decided), + Err(refusal) => { + let stored: Option = row.get_by_name("stream_specification").ok().flatten(); + let current: Option = stored + .as_deref() + .map(serde_json::from_str) + .transpose() + .map_err(internal)?; + let label: Option = row.get_by_name("stream_label").ok().flatten(); + if let Some(label) = + disabled_stream_to_close(current.as_ref(), label.as_deref(), requested) + { + self.close_stream_shards(account_ks, table_id, label) + .await?; + } + Err(refusal) + } + } + } + /// The refusal for a stream change whose conditional catalog write found /// the stream fields changed: re-read the row and decide the request /// against it, which refuses the loser of two overlapping changes with the diff --git a/crates/storage-cassandra/src/update_table.rs b/crates/storage-cassandra/src/update_table.rs index ceb28b703..a0c438507 100644 --- a/crates/storage-cassandra/src/update_table.rs +++ b/crates/storage-cassandra/src/update_table.rs @@ -156,8 +156,15 @@ impl CassandraEngine { // in a different keyspace. let mut pending_stream = None; if let Some(spec) = &input.stream_specification { - let (cols, pending) = - crate::stream_lifecycle::PendingStream::decide(&row, &input.table_name, spec)?; + let (cols, pending) = self + .decide_stream_change_or_close( + &row, + &self.account_keyspace(account_id), + &table_id, + &input.table_name, + spec, + ) + .await?; for (col, val) in cols { table_cols.push((col, Value::from(val.as_str()))); } diff --git a/crates/storage-mongodb/src/stream_lifecycle.rs b/crates/storage-mongodb/src/stream_lifecycle.rs index 42c01b436..dfa658f64 100644 --- a/crates/storage-mongodb/src/stream_lifecycle.rs +++ b/crates/storage-mongodb/src/stream_lifecycle.rs @@ -20,8 +20,9 @@ use mongodb::options::FindOptions; use extenddb_core::types::{StreamSpecification, StreamViewType}; use extenddb_storage::error::StorageError; use extenddb_storage::stream_lineage::{ - SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, enable_still_current, - next_stream_label, open_shard_ids_of, shard_id, stored_specification, stream_change, + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, + enable_still_current, next_stream_label, open_shard_ids_of, shard_id, stored_specification, + stream_change, }; use crate::MongoEngine; @@ -231,6 +232,34 @@ impl MongoEngine { }) } + /// [`Self::prepare_stream_update`], and for a disable refused because the + /// stored specification is already disabled, the close of the current + /// label's open shards first (left by a release that did not close them, + /// or by a close that failed after the catalog write); the refusal is + /// still the answer. + pub(crate) async fn prepare_stream_update_or_close( + &self, + update_doc: &mut Document, + table_doc: &Document, + table_name: &str, + requested: &StreamSpecification, + ) -> Result { + match Self::prepare_stream_update(update_doc, table_doc, table_name, requested) { + Ok(pending) => Ok(pending), + Err(refusal) => { + let table = table_row(table_name, table_doc); + if let Some(label) = disabled_stream_to_close( + table.stream_specification.as_ref(), + table.stream_label.as_deref(), + requested, + ) { + self.close_stream_shards(&table.table_id, label).await?; + } + Err(refusal) + } + } + } + /// The table's stream fields and its document, by name. pub(crate) async fn stream_table_row( &self, diff --git a/crates/storage-mongodb/src/table_engine.rs b/crates/storage-mongodb/src/table_engine.rs index 6ca721b7d..7ea77c030 100644 --- a/crates/storage-mongodb/src/table_engine.rs +++ b/crates/storage-mongodb/src/table_engine.rs @@ -839,12 +839,15 @@ impl MongoEngine { .await .map_err(|e| StorageError::Internal(e.to_string()))? .ok_or_else(|| StorageError::TableNotFound(input.table_name.clone()))?; - pending_stream = Some(Self::prepare_stream_update( - &mut update_doc, - &table_doc, - &input.table_name, - ss, - )?); + pending_stream = Some( + self.prepare_stream_update_or_close( + &mut update_doc, + &table_doc, + &input.table_name, + ss, + ) + .await?, + ); for field in ["stream_specification", "stream_label"] { stream_guard.insert( field, diff --git a/crates/storage-postgres/src/stream_lifecycle.rs b/crates/storage-postgres/src/stream_lifecycle.rs index b5b038672..553fedaaf 100644 --- a/crates/storage-postgres/src/stream_lifecycle.rs +++ b/crates/storage-postgres/src/stream_lifecycle.rs @@ -16,8 +16,9 @@ use extenddb_core::types::{StreamSpecification, StreamViewType}; use extenddb_storage::error::StorageError; use extenddb_storage::stream_lineage::{ - SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, enable_still_current, - next_stream_label, open_shard_ids_of, shard_id, stored_specification, stream_change, + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, + enable_still_current, next_stream_label, open_shard_ids_of, shard_id, stored_specification, + stream_change, }; use sqlx::PgPool; @@ -223,8 +224,15 @@ impl PostgresEngine { /// with [`PendingStream::apply`] once the catalog has committed: a request /// refused after this point (an invalid index update in the same call) /// then leaves no shards behind. + /// + /// A disable refused because the stored specification is already disabled + /// first closes the current label's open shards in the data database + /// (left by a release that did not close them, or by a close that failed + /// after its catalog write), then returns the refusal; the catalog + /// transaction has nothing to commit. pub(crate) async fn apply_stream_specification( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + data_pool: &PgPool, account_id: &str, table_name: &str, table_id: &str, @@ -249,7 +257,17 @@ impl PostgresEngine { .await .map_err(internal)? .flatten(); - let change = stream_change(table_name, current.as_ref(), requested)?; + let change = match stream_change(table_name, current.as_ref(), requested) { + Ok(change) => change, + Err(refusal) => { + if let Some(label) = + disabled_stream_to_close(current.as_ref(), current_label.as_deref(), requested) + { + Self::close_stream_shards(data_pool, table_id, label).await?; + } + return Err(refusal); + } + }; let stored = serde_json::to_value(stored_specification(change, current.as_ref(), requested)) .map_err(internal)?; diff --git a/crates/storage-postgres/src/update_table.rs b/crates/storage-postgres/src/update_table.rs index 61b4174ef..b45962b18 100755 --- a/crates/storage-postgres/src/update_table.rs +++ b/crates/storage-postgres/src/update_table.rs @@ -337,6 +337,7 @@ impl PostgresEngine { Some(spec) => Some( Self::apply_stream_specification( &mut tx, + &self.data_pool, account_id, &input.table_name, &table_id, diff --git a/crates/storage-postgres/tests/stream_disable.rs b/crates/storage-postgres/tests/stream_disable.rs index 41fe212c5..ea46a8120 100644 --- a/crates/storage-postgres/tests/stream_disable.rs +++ b/crates/storage-postgres/tests/stream_disable.rs @@ -542,3 +542,113 @@ async fn a_stream_from_before_v0_1_14_survives_the_upgrade() { ); s.cleanup().await; } + +/// A stream disabled by a release that stored `{StreamEnabled: false}` and +/// left its four name-based shards open (the same state a close that failed +/// after its catalog write leaves): writes still record into it. A disable is +/// refused, as on the service, and closes the open shards first, so the next +/// write records nothing. +#[tokio::test] +async fn a_disable_refused_on_a_disabled_stream_closes_the_shards_it_left_open() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("base", false)) + .await + .expect("create"); + activate(&s, "base").await; + let table_id = describe_table(&s, "base").await.table_id; + let label = "2026-10-20T10:00:00.000"; + sqlx::query( + "UPDATE tables SET stream_label = $1, stream_specification = $2 \ + WHERE account_id = $3 AND table_name = 'base'", + ) + .bind(label) + .bind(serde_json::json!({"StreamEnabled": false})) + .bind(ACCOUNT) + .execute(&s.db) + .await + .expect("seed the catalog row"); + for i in 0..4 { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES ($1, $2, $3)", + ) + .bind(format!("shardId-base-{i:016}")) + .bind(&table_id) + .bind(format!("{:021}", 0)) + .execute(&s.db) + .await + .expect("seed an open shard"); + } + let open_shards = || async { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM stream_shards WHERE ending_sequence_number IS NULL", + ) + .fetch_one(&s.db) + .await + .expect("count open shards") + }; + let arn = describe_table(&s, "base") + .await + .latest_stream_arn + .expect("the seeded stream arn"); + assert!(matches!( + describe_stream(&s, &arn).await.stream_status, + StreamStatus::Disabled + )); + put(&s, "base", "dark", StreamViewType::KeysOnly).await; + assert_eq!(record_count(&s).await, 1, "the open shards still record"); + + match set_stream(&s, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!( + open_shards().await, + 0, + "the refused disable closed the shards" + ); + let closed = describe_stream(&s, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + for shard in &closed.shards { + let expected = last_record(&s, &shard.shard_id) + .await + .unwrap_or_else(|| shard.sequence_number_range.starting_sequence_number.clone()); + assert_eq!( + shard + .sequence_number_range + .ending_sequence_number + .as_deref(), + Some(expected.as_str()) + ); + } + put(&s, "base", "after", StreamViewType::KeysOnly).await; + assert_eq!( + record_count(&s).await, + 1, + "a write after the close records nothing" + ); + // The stored specification is untouched, and a second refusal has nothing + // left to close. + assert_eq!( + describe_table(&s, "base").await.stream_specification, + Some(StreamSpecification { + stream_enabled: false, + stream_view_type: None, + }) + ); + match set_stream(&s, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!(open_shards().await, 0); + s.cleanup().await; +} diff --git a/crates/storage-sqlite/src/stream_lifecycle.rs b/crates/storage-sqlite/src/stream_lifecycle.rs index d67f4045c..155acad1b 100644 --- a/crates/storage-sqlite/src/stream_lifecycle.rs +++ b/crates/storage-sqlite/src/stream_lifecycle.rs @@ -12,8 +12,8 @@ use extenddb_core::types::{StreamSpecification, StreamViewType}; use extenddb_storage::error::StorageError; use extenddb_storage::stream_lineage::{ - SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, next_stream_label, shard_id, - stored_specification, stream_change, + SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, + next_stream_label, shard_id, stored_specification, stream_change, }; use crate::store::SqliteEngine; @@ -105,33 +105,51 @@ impl SqliteEngine { /// Apply an `UpdateTable` `StreamSpecification` within the caller's /// transaction. /// - /// Disabling clears the stored specification, so DescribeTable omits - /// `StreamSpecification`, keeps `stream_label` (`LatestStreamArn` keeps - /// naming the disabled stream) and closes the stream's shards. Enabling - /// closes any shard still open (a stream disabled before shards were - /// closed on disable) and creates a new stream. + /// Disabling stores `{StreamEnabled: false}` with the stream's view type, + /// which DescribeTable omits, keeps `stream_label` (`LatestStreamArn` + /// keeps naming the disabled stream) and closes the stream's shards. + /// Enabling closes any shard still open (a stream disabled before shards + /// were closed on disable) and creates a new stream. + /// + /// `Ok(Some(refusal))` is a disable refused because the stored + /// specification is already disabled, whose open shards (left by a release + /// that did not close them, or by a close that failed after its catalog + /// write) this call closed in the transaction: the caller commits it and + /// answers the refusal. Any other refusal is `Err`, with nothing to commit. pub(crate) async fn apply_stream_specification( tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, account_id: &str, table_name: &str, table_id: &str, requested: &StreamSpecification, - ) -> Result<(), StorageError> { - let current: Option = sqlx::query_scalar( - "SELECT stream_specification FROM tables WHERE account_id = ? AND table_name = ?", + ) -> Result, StorageError> { + let row: Option<(Option, Option)> = sqlx::query_as( + "SELECT stream_specification, stream_label FROM tables \ + WHERE account_id = ? AND table_name = ?", ) .bind(account_id) .bind(table_name) .fetch_optional(&mut **tx) .await - .map_err(internal)? - .flatten(); + .map_err(internal)?; + let (current, current_label) = row.unwrap_or_default(); let current: Option = current .as_deref() .map(serde_json::from_str) .transpose() .map_err(internal)?; - let change = stream_change(table_name, current.as_ref(), requested)?; + let change = match stream_change(table_name, current.as_ref(), requested) { + Ok(change) => change, + Err(refusal) => { + if disabled_stream_to_close(current.as_ref(), current_label.as_deref(), requested) + .is_some() + { + Self::close_stream_shards(tx, table_id).await?; + return Ok(Some(refusal)); + } + return Err(refusal); + } + }; let stored = serde_json::to_string(&stored_specification(change, current.as_ref(), requested)) .map_err(internal)?; @@ -148,7 +166,7 @@ impl SqliteEngine { if let StreamChange::Enable(view_type) = change { Self::init_stream_shards(tx, account_id, table_name, table_id, view_type).await?; } - Ok(()) + Ok(None) } /// The table's stream columns and its key schema JSON, by name. diff --git a/crates/storage-sqlite/src/stream_lifecycle_tests.rs b/crates/storage-sqlite/src/stream_lifecycle_tests.rs index c470b96fb..79db3195e 100644 --- a/crates/storage-sqlite/src/stream_lifecycle_tests.rs +++ b/crates/storage-sqlite/src/stream_lifecycle_tests.rs @@ -355,3 +355,128 @@ async fn a_stream_from_before_labelled_shard_ids_closes_and_is_replaced() { assert_eq!(new.shards.len(), 4); engine.pool.close().await; } + +/// A stream disabled by a release that stored `{StreamEnabled: false}` and +/// left its four shards open (the same state a close that failed after its +/// catalog write leaves): writes still record into it. A disable is refused, +/// as on the service, and closes the open shards first, so the next write +/// records nothing; an enable on that state closes them too. +#[tokio::test] +async fn a_disable_refused_on_a_disabled_stream_closes_the_shards_it_left_open() { + let engine = engine().await; + create(&engine, "base", true).await; + let arn = describe_table(&engine, "base") + .await + .latest_stream_arn + .unwrap(); + // The shards in the shape the earlier release created them, and what its + // disable wrote: the specification alone. + sqlx::query( + "UPDATE stream_shards SET shard_id = 'shardId-' || table_id || '-' || \ + printf('%016d', CAST(substr(shard_id, -2) AS INTEGER)) \ + WHERE table_id = (SELECT table_id FROM tables WHERE table_name = 'base')", + ) + .execute(&engine.pool) + .await + .unwrap(); + put(&engine, "base", "before", StreamViewType::NewAndOldImages).await; + sqlx::query( + "UPDATE tables SET stream_specification = '{\"StreamEnabled\":false}' \ + WHERE table_name = 'base'", + ) + .execute(&engine.pool) + .await + .unwrap(); + let open_shards = || async { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM stream_shards WHERE ending_sequence_number IS NULL", + ) + .fetch_one(&engine.pool) + .await + .unwrap() + }; + assert_eq!(open_shards().await, 4); + let described = describe_stream(&engine, &arn).await; + assert!(matches!(described.stream_status, StreamStatus::Disabled)); + put(&engine, "base", "dark", StreamViewType::NewAndOldImages).await; + let recorded = || async { + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM stream_records") + .fetch_one(&engine.pool) + .await + .unwrap() + }; + assert_eq!(recorded().await, 2, "the open shards still record"); + + match set_stream(&engine, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!( + open_shards().await, + 0, + "the refused disable closed the shards" + ); + let closed = describe_stream(&engine, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!(closed.stream_view_type, StreamViewType::KeysOnly); + for shard in &closed.shards { + let last = records_in(&engine, &shard.shard_id).await.pop(); + let expected = + last.unwrap_or_else(|| shard.sequence_number_range.starting_sequence_number.clone()); + assert_eq!( + shard + .sequence_number_range + .ending_sequence_number + .as_deref(), + Some(expected.as_str()) + ); + } + put(&engine, "base", "after", StreamViewType::NewAndOldImages).await; + assert_eq!( + recorded().await, + 2, + "a write after the close records nothing" + ); + // A second refusal has nothing left to close and changes nothing. + match set_stream(&engine, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!(open_shards().await, 0); + + // The same state, met by an enable: the enable closes the open shards. + create(&engine, "base2", true).await; + sqlx::query( + "UPDATE tables SET stream_specification = '{\"StreamEnabled\":false}' \ + WHERE table_name = 'base2'", + ) + .execute(&engine.pool) + .await + .unwrap(); + assert_eq!(open_shards().await, 4); + set_stream( + &engine, + "base2", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + assert_eq!( + open_shards().await, + 4, + "only the new stream's shards are open" + ); + let closed_by_enable: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_shards s JOIN tables t ON t.table_id = s.table_id \ + WHERE t.table_name = 'base2' AND s.ending_sequence_number IS NOT NULL", + ) + .fetch_one(&engine.pool) + .await + .unwrap(); + assert_eq!(closed_by_enable, 4); + engine.pool.close().await; +} diff --git a/crates/storage-sqlite/src/update_table.rs b/crates/storage-sqlite/src/update_table.rs index 4677eef49..67ad6ca84 100644 --- a/crates/storage-sqlite/src/update_table.rs +++ b/crates/storage-sqlite/src/update_table.rs @@ -311,15 +311,23 @@ impl SqliteEngine { ) .await?; } - if let Some(spec) = &input.stream_specification { - Self::apply_stream_specification( + if let Some(spec) = &input.stream_specification + && let Some(refusal) = Self::apply_stream_specification( &mut tx, account_id, &input.table_name, &table_id, spec, ) - .await?; + .await? + { + // A disable of an already disabled stream: the request is refused, + // and the transaction holds only the close of the shards the first + // disable left open (a stream change travels alone in the request). + tx.commit() + .await + .map_err(|e| StorageError::Internal(e.to_string()))?; + return Err(refusal); } // GSI create/delete. diff --git a/crates/storage/src/stream_lineage.rs b/crates/storage/src/stream_lineage.rs index 99676074c..c40c9b56c 100644 --- a/crates/storage/src/stream_lineage.rs +++ b/crates/storage/src/stream_lineage.rs @@ -104,6 +104,30 @@ pub fn stream_change( } } +/// The stream a refused disable closes before it is answered: the table's +/// current label, when `requested` disables a table whose stored specification +/// is already disabled. +/// +/// A release before shards were closed on disable left a disabled stream's +/// four shards open, and a close that fails after its catalog write leaves the +/// same state until the request is retried; either way `DescribeStream` +/// reports `DISABLED` while writes keep recording into the open shards. The +/// retry is refused with [`stream_change`]'s sentence, as on the service, and +/// the backend closes the current label's open shards first, which is what the +/// first disable would have done. `None` when the request enables, when the +/// table never had a stream (no stored specification), or when it has no +/// label. +#[must_use] +pub fn disabled_stream_to_close<'a>( + current: Option<&StreamSpecification>, + current_label: Option<&'a str>, + requested: &StreamSpecification, +) -> Option<&'a str> { + (!requested.stream_enabled && current.is_some_and(|c| !c.stream_enabled)) + .then_some(current_label) + .flatten() +} + /// The specification to store once a stream change is decided: the request's /// own on an enable, and on a disable `{StreamEnabled: false}` carrying the /// closed stream's view type, so a legacy stream (whose view type lives only diff --git a/crates/storage/src/stream_lineage_tests.rs b/crates/storage/src/stream_lineage_tests.rs index 917e5c062..d99b7f718 100644 --- a/crates/storage/src/stream_lineage_tests.rs +++ b/crates/storage/src/stream_lineage_tests.rs @@ -79,6 +79,44 @@ fn the_two_refusals_are_the_measured_sentences() { ); } +#[test] +fn a_refused_disable_of_a_disabled_stream_closes_the_current_label_first() { + let label = "2026-10-20T10:00:00.000"; + let disabled = spec(false, Some(StreamViewType::NewImage)); + let disable = spec(false, None); + let enable = spec(true, Some(StreamViewType::KeysOnly)); + // The stored specification says disabled: the first disable (an earlier + // release's, or one whose close failed) may have left the shards open. + assert_eq!( + disabled_stream_to_close(Some(&disabled), Some(label), &disable), + Some(label) + ); + assert_eq!( + disabled_stream_to_close(Some(&spec(false, None)), Some(label), &disable), + Some(label) + ); + // A table that never had a stream, or has no label, has nothing to close; + // an enabled stream is closed by the disable itself; an enable closes + // through its own change. + assert_eq!(disabled_stream_to_close(None, Some(label), &disable), None); + assert_eq!( + disabled_stream_to_close(Some(&disabled), None, &disable), + None + ); + assert_eq!( + disabled_stream_to_close( + Some(&spec(true, Some(StreamViewType::KeysOnly))), + Some(label), + &disable + ), + None + ); + assert_eq!( + disabled_stream_to_close(Some(&disabled), Some(label), &enable), + None + ); +} + #[test] fn labels_round_trip_through_shard_ids() { assert_eq!(label_millis("1970-01-01T00:00:00.001"), Some(1)); From bdba4fb7859b451cc4171fc92550910bf465dd50 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:17:41 +0000 Subject: [PATCH 07/15] perf(postgres): lock only the record's shard row on a streamed write Every write to a streamed table locked all four open shard rows FOR SHARE, so concurrent writers on one table kept extending the same MultiXacts for a record that goes into one shard. The open set is now read without a lock, and the chosen row alone is locked FOR KEY SHARE with its ending_sequence_number re-checked. The close still locks the open rows FOR UPDATE before taking the ending numbers, which conflicts with this lock, so a write either commits before the ending number is taken or finds its row closed and records nothing. A storage test runs eight writers against a disable and checks that no record is past its shard's ending number. --- .../storage-postgres/src/data/tx_helpers.rs | 26 ++++- .../storage-postgres/tests/stream_disable.rs | 97 +++++++++++++++++++ 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/crates/storage-postgres/src/data/tx_helpers.rs b/crates/storage-postgres/src/data/tx_helpers.rs index a77de1cb6..f406c0d90 100755 --- a/crates/storage-postgres/src/data/tx_helpers.rs +++ b/crates/storage-postgres/src/data/tx_helpers.rs @@ -321,14 +321,18 @@ pub(super) async fn write_stream_record_in_tx( // disabled stream's shards are closed and take no new records, so a write // that read the table's key info before a disable records nothing. // - // FOR SHARE: a disable locks the same rows FOR UPDATE before it takes the - // ending numbers, so this write either commits before the ending number - // is taken or, waiting here, re-evaluates `ending_sequence_number IS NULL` - // on the closed rows and records nothing (`close_stream_shards`). + // The open set is read without a lock, and only the shard this record + // goes to is locked, FOR KEY SHARE, re-checking that it is still open: a + // disable locks the open rows FOR UPDATE before it takes the ending + // numbers (`close_stream_shards`), which conflicts with this lock, so the + // write either commits before the ending number is taken or, waiting + // here, finds the row closed and records nothing. Locking every open row + // FOR SHARE made concurrent writers to one table extend the same + // MultiXacts on all four rows for a record that goes into one. let shards: Vec<(String,)> = sqlx::query_as( "SELECT shard_id FROM stream_shards \ WHERE table_id = $1 AND ending_sequence_number IS NULL \ - ORDER BY shard_id FOR SHARE", + ORDER BY shard_id", ) .bind(&key_info.table_id) .fetch_all(&mut **tx) @@ -344,6 +348,18 @@ pub(super) async fn write_stream_record_in_tx( #[allow(clippy::cast_possible_truncation)] let idx = (hash as usize) % shards.len(); let shard_id = &shards[idx].0; + let still_open: Option<(String,)> = sqlx::query_as( + "SELECT shard_id FROM stream_shards \ + WHERE shard_id = $1 AND ending_sequence_number IS NULL FOR KEY SHARE", + ) + .bind(shard_id) + .fetch_optional(&mut **tx) + .await + .map_err(db_error)?; + if still_open.is_none() { + // Closed between the read and the lock: the stream is disabled. + return Ok(()); + } // Generate monotonic sequence number within the transaction (CB-21). let (seq_val,): (i64,) = sqlx::query_as("SELECT nextval('stream_seq')") diff --git a/crates/storage-postgres/tests/stream_disable.rs b/crates/storage-postgres/tests/stream_disable.rs index ea46a8120..5ed84fbd0 100644 --- a/crates/storage-postgres/tests/stream_disable.rs +++ b/crates/storage-postgres/tests/stream_disable.rs @@ -652,3 +652,100 @@ async fn a_disable_refused_on_a_disabled_stream_closes_the_shards_it_left_open() assert_eq!(open_shards().await, 0); s.cleanup().await; } + +/// Writers racing a disable: every record a write lands is in the closed +/// stream at or below its shard's `EndingSequenceNumber`, or the write records +/// nothing. The write path locks only its own shard row, `FOR KEY SHARE`, and +/// the close locks the open rows `FOR UPDATE` before taking the ending +/// numbers, so a record is never written past them. +#[tokio::test] +async fn a_write_racing_a_disable_lands_before_the_ending_number_or_not_at_all() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("race", true)) + .await + .expect("create"); + activate(&s, "race").await; + let key_info = s + .engine + .table_key_info(ACCOUNT, "race") + .await + .expect("key info"); + let arn = describe_table(&s, "race") + .await + .latest_stream_arn + .expect("stream arn"); + + let mut writers = tokio::task::JoinSet::new(); + for w in 0..8u32 { + let engine = s.engine.clone(); + let key_info = key_info.clone(); + writers.spawn(async move { + let mut acked = 0u32; + let mut failed = 0u32; + for i in 0..40u32 { + let item = + serde_json::from_value(serde_json::json!({"pk": {"S": format!("w{w}-{i}")}})) + .expect("item"); + let capture = StreamCapture { + view_type: StreamViewType::NewAndOldImages, + user_identity: None, + region: REGION.into(), + }; + match engine + .put_item( + &key_info, + item, + false, + None, + &ExpressionMaps::default(), + Some(&capture), + ) + .await + { + Ok(_) => acked += 1, + Err(_) => failed += 1, + } + } + (acked, failed) + }); + } + tokio::time::sleep(std::time::Duration::from_millis(40)).await; + set_stream(&s, "race", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let (mut acked, mut failed) = (0u32, 0u32); + while let Some(done) = writers.join_next().await { + let (a, f) = done.expect("writer task"); + acked += a; + failed += f; + } + assert_eq!(acked + failed, 320); + + let closed = describe_stream(&s, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert!( + closed + .shards + .iter() + .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) + ); + let past_the_end: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_records r JOIN stream_shards s ON s.shard_id = r.shard_id \ + WHERE s.ending_sequence_number IS NULL OR r.sequence_number > s.ending_sequence_number", + ) + .fetch_one(&s.db) + .await + .expect("count records past an ending number"); + assert_eq!(past_the_end, 0, "{acked} acked, {failed} failed"); + let recorded = record_count(&s).await; + assert!( + recorded >= 1 && recorded <= i64::from(acked), + "{recorded} records for {acked} acked writes" + ); + s.cleanup().await; +} From b3eae75271389b03c87ae08249f710a497c66ee4 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:23:26 +0000 Subject: [PATCH 08/15] perf(mongodb): index stream_shards by table_id and shard_id stream_shards had the unique shard_id index alone, so the open-shard read on every write to a streamed table, and the shard reads behind DescribeStream, GetShardIterator and ListStreams, scanned the whole collection across all accounts: with about 20k shard documents a write examined 20,008 to return 4, and PutItem p50 went from 7.9 ms to 40 ms (measured in review). Each enable now adds four documents that stay, so the collection grows with every disable and enable cycle. A {table_id: 1, shard_id: 1} index covers the filter and the sort of those reads. extenddb init creates it in create_data_db. MongoDB's data migration, which extenddb migrate --yes runs on every upgrade, creates it too, so an existing deployment gets it; creating it again is a no-op, and the migration reports it pending while it is missing. A live test drops the index, checks it is reported pending, runs the migration and finds it. --- crates/storage-mongodb/src/bootstrapper.rs | 125 +++++++++++++++++++-- 1 file changed, 118 insertions(+), 7 deletions(-) diff --git a/crates/storage-mongodb/src/bootstrapper.rs b/crates/storage-mongodb/src/bootstrapper.rs index c37c3f36f..91c60220a 100644 --- a/crates/storage-mongodb/src/bootstrapper.rs +++ b/crates/storage-mongodb/src/bootstrapper.rs @@ -12,6 +12,34 @@ use extenddb_storage::bootstrapper::{AdminBootstrapResult, Bootstrapper}; use extenddb_storage::error::StorageError; use extenddb_storage::management_store::{OpError, OpResult}; +/// The name of the `{table_id: 1, shard_id: 1}` index on `stream_shards` +/// (the name `MongoDB` gives those keys by default). +const STREAM_SHARDS_TABLE_INDEX: &str = "table_id_1_shard_id_1"; + +/// Create the `{table_id: 1, shard_id: 1}` index on `stream_shards`. +/// +/// Every write to a streamed table selects the table's open shards by +/// `table_id` and sorts them by `shard_id`, and `DescribeStream`, +/// `GetShardIterator` and `ListStreams` read a table's shard rows the same +/// way. Without it those reads scan the whole collection across all +/// accounts, and each enable adds four documents that are never removed. +async fn create_stream_shards_table_index(db: &mongodb::Database) -> OpResult<()> { + db.collection::("stream_shards") + .create_index( + IndexModel::builder() + .keys(doc! { "table_id": 1, "shard_id": 1 }) + .options( + IndexOptions::builder() + .name(STREAM_SHARDS_TABLE_INDEX.to_owned()) + .build(), + ) + .build(), + ) + .await + .map_err(|e| OpError::Internal(format!("stream_shards table_id index: {e}")))?; + Ok(()) +} + /// `MongoDB` bootstrapper for init/destroy/migrate operations. pub struct MongoBootstrapper { client: mongodb::Client, @@ -129,8 +157,8 @@ impl Bootstrapper for MongoBootstrapper { // stream_shards: unique index on shard_id so a subsequent init/ // recreate can never insert a duplicate shard document under the // same shard_id. Combined with `table_id`-derived shard_ids - // (see stream_engine::build_shard_id) this rules out cross-tenant - // shard collisions structurally. + // (see extenddb_storage::stream_lineage::shard_id) this rules out + // cross-tenant shard collisions structurally. db.create_collection("stream_shards") .await .map_err(|e| OpError::Internal(format!("Failed to create stream_shards: {e}")))?; @@ -143,6 +171,7 @@ impl Bootstrapper for MongoBootstrapper { ) .await .map_err(|e| OpError::Internal(format!("stream_shards shard_id index: {e}")))?; + create_stream_shards_table_index(&db).await?; // stream_records: TTL index enforcing DDB's 24-hour retention. // The `expireAfterSeconds` is the delta from the field value, @@ -378,15 +407,38 @@ impl Bootstrapper for MongoBootstrapper { async fn run_data_migrations(&self) -> OpResult<()> { // Data database schema is minimal for MongoDB (just idempotency_tokens) - // Table collections are created on-demand - Ok(()) + // Table collections are created on-demand. The one index added to a + // collection create_data_db makes, `stream_shards` by + // `{table_id, shard_id}`, is created here too, so a deployment + // initialised before it gets it from `extenddb migrate --yes`; + // creating an index that exists with the same keys and name is a + // no-op. + create_stream_shards_table_index(&self.data_db()).await } async fn pending_data_migrations(&self) -> OpResult> { - // MongoDB has no versioned data-database migrations — table + // MongoDB has no versioned data-database migrations: table // collections are created on-demand, and the idempotency_tokens - // collection is created in create_data_db. Nothing to apply. - Ok(Vec::new()) + // collection is created in create_data_db. The `stream_shards` + // `{table_id, shard_id}` index is pending while it is missing. + let names = match self + .data_db() + .collection::("stream_shards") + .list_index_names() + .await + { + Ok(names) => names, + // NamespaceNotFound: no `stream_shards` collection yet. + Err(e) if matches!(*e.kind, mongodb::error::ErrorKind::Command(ref c) if c.code == 26) => { + Vec::new() + } + Err(e) => return Err(OpError::Internal(format!("stream_shards indexes: {e}"))), + }; + Ok(if names.iter().any(|n| n == STREAM_SHARDS_TABLE_INDEX) { + Vec::new() + } else { + vec![format!("stream_shards index {STREAM_SHARDS_TABLE_INDEX}")] + }) } async fn record_data_connection(&self) -> OpResult<()> { @@ -670,6 +722,65 @@ fn redact_connection_string(uri: &str) -> String { mod tests { use super::redact_connection_string; + /// A data database created before the `stream_shards` + /// `{table_id, shard_id}` index: the data migration `extenddb migrate + /// --yes` runs reports it pending, creates it, and is a no-op the second + /// time. Uses the `extenddb_data` database of the server in + /// `EXTENDDB_TEST_MONGODB_URI`, as `extenddb init` does. + #[tokio::test] + #[ignore = "needs a MongoDB server in EXTENDDB_TEST_MONGODB_URI"] + async fn live_stream_shards_index_comes_from_the_data_migration() { + use bson::{Document, doc}; + use extenddb_storage::bootstrapper::Bootstrapper; + + let uri = std::env::var("EXTENDDB_TEST_MONGODB_URI") + .expect("set EXTENDDB_TEST_MONGODB_URI to a MongoDB server"); + let client = crate::connect_guarded(&uri, None, false) + .await + .expect("connect"); + let boot = super::MongoBootstrapper { + client, + connection_string: uri, + }; + let shards = boot.data_db().collection::("stream_shards"); + // The collection as an earlier init left it: the unique shard_id + // index and no {table_id, shard_id} index. + let _ = boot.data_db().create_collection("stream_shards").await; + let _ = shards.drop_index(super::STREAM_SHARDS_TABLE_INDEX).await; + let names = shards.list_index_names().await.expect("index names"); + assert!(!names.iter().any(|n| n == super::STREAM_SHARDS_TABLE_INDEX)); + assert_eq!( + boot.pending_data_migrations().await.expect("pending"), + vec![format!( + "stream_shards index {}", + super::STREAM_SHARDS_TABLE_INDEX + )] + ); + + boot.run_data_migrations().await.expect("data migration"); + let indexes: Vec = { + use futures::TryStreamExt; + shards + .list_indexes() + .await + .expect("list indexes") + .map_ok(|m| m.keys) + .try_collect() + .await + .expect("index keys") + }; + assert!(indexes.contains(&doc! { "table_id": 1, "shard_id": 1 })); + assert!( + boot.pending_data_migrations() + .await + .expect("pending") + .is_empty() + ); + boot.run_data_migrations() + .await + .expect("the second run is a no-op"); + } + #[test] fn redact_userinfo_with_password() { assert_eq!( From 7823db82a87320ca4a5882ba74f8fc97c20b5fad Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:30:01 +0000 Subject: [PATCH 09/15] fix(cassandra): start a new stream above the closed streams' ending numbers A new stream's shards started at a value drawn from the enabling server's clock alone. With several servers, a record of the closed stream written by a server whose clock is ahead could sit above that start, and so read in both the closed stream and the new one. The start is now one above the larger of this server's next clock value and the last ending number among the table's shards, taken after the close. A server whose clock is behind the start can still write a new-stream record below it, which no stream reads, and on several servers a raced record from a server whose clock is ahead can still land above the start; the module and the design document note both cases beside the deferred cross-instance item. The design document also said the new shard starts where the closed one ended, which neither the code nor its later paragraph did. A close writes a shard's ending number only when the shard has none (UPDATE ... IF ending_sequence_number = null). An open shard's range has no upper bound, so a disable whose close read the old shards as open and took their ending numbers after an overlapping enable had closed them and started the next stream would otherwise set them to a record of the new stream from the shared partition, above the new stream's start. Cassandra direct tests: one moves a closed stream's ending numbers an hour ahead of the clock and checks that the new stream starts above them; one runs a close with the old stream's rows as read before the disable, after the enable and eight writes to the new stream, and checks that the old shards keep their ending numbers. --- .../storage-cassandra/src/stream_lifecycle.rs | 83 +++++-- .../tests/direct/stream_disable.rs | 229 ++++++++++++++++++ docs/design/07-component-streams.md | 4 +- 3 files changed, 299 insertions(+), 17 deletions(-) diff --git a/crates/storage-cassandra/src/stream_lifecycle.rs b/crates/storage-cassandra/src/stream_lifecycle.rs index 7b3a68344..d23b5f039 100644 --- a/crates/storage-cassandra/src/stream_lifecycle.rs +++ b/crates/storage-cassandra/src/stream_lifecycle.rs @@ -12,10 +12,19 @@ //! stored in one partition per shard index, `shardId-{table_id}-{index:012}`, //! whatever stream is current, and a stream's shard is a range of that //! partition: the records after its `StartingSequenceNumber`, up to its -//! `EndingSequenceNumber` once closed. A new stream's shard starts at a clock -//! value drawn when the stream is enabled, above every sequence issued before, -//! so a record belongs to at most one stream. The first stream of a table -//! starts at zero. +//! `EndingSequenceNumber` once closed. A new stream's shards start one above +//! the larger of this server's next clock value and the last ending number +//! among the table's shards, drawn when the stream is enabled, so a record +//! belongs to at most one stream. The first stream of a table starts at zero. +//! +//! With several servers, the sequence a write draws is its own server's clock +//! (`stream_util::HybridClock`), while the start is taken on the server that +//! runs the enable. Taking the closed shards' ending numbers into the start +//! keeps a record of the old stream, written by a server whose clock is ahead, +//! from reading in both streams. A server whose clock is behind the start can +//! still write a new-stream record below it, which no stream reads; that case +//! stays with the deferred cross-instance consistency item in the design +//! document. use cdrs_tokio::types::IntoRustByName; use extenddb_core::types::{StreamSpecification, StreamViewType}; @@ -44,6 +53,14 @@ pub(crate) fn physical_shard_id(table_id: &str, shard_id: &str) -> String { ) } +/// The sequence number one above `sequence`, in its 23-digit shape. +fn next_sequence(sequence: &str) -> Result { + let n: u128 = sequence + .parse() + .map_err(|_| StorageError::Internal(format!("malformed sequence number {sequence}")))?; + Ok(format!("{:0width$}", n + 1, width = sequence.len())) +} + /// The table id a shard id names (`shardId-{table_id}-{digits}`). pub(crate) fn shard_table_id(shard_id: &str) -> Option<&str> { shard_id @@ -100,7 +117,10 @@ impl PendingStream { /// after its inserts and, when the row no longer says enabled with this /// label, closes the shards it inserted. A disable written before the /// re-read is seen by it; one written after finds all four rows open and - /// closes them itself; the two closes carry the same ending number. + /// closes them itself. A shard keeps the ending number of the first close + /// that writes one (`close_stream_shards`), so a disable whose close runs + /// after an overlapping enable started the next stream cannot move the + /// closed shard's ending number up into the new stream's records. pub(crate) async fn apply( self, engine: &CassandraEngine, @@ -117,18 +137,28 @@ impl PendingStream { let Some((label, view_type)) = self.open else { return Ok(()); }; - // Each new shard starts at a sequence number drawn now from the same - // clock the writers draw from, so it is above every sequence issued so - // far. A write of the closed stream that commits after the close (its - // LOGGED BATCH drew its sequence before the ending number was taken) - // then falls between the old shard's ending number and the new + // Each new shard starts one above the larger of a sequence number + // drawn now from the clock this server's writers draw from and the + // last ending number among the table's shards (the ones just closed + // included), so it is above every sequence issued so far on this + // server and every record the closed streams hold, whichever server + // wrote it. A write of the closed stream that commits after the close + // (its LOGGED BATCH drew its sequence before the ending number was + // taken) then falls between the old shard's ending number and the new // shard's starting number: in neither stream, rather than in the new // one under the old view type. - let start = engine + let local = engine .hlc .lock() .unwrap_or_else(std::sync::PoisonError::into_inner) .generate(); + let last_end = engine + .stream_shard_rows(account_ks, Some(table_id)) + .await? + .into_iter() + .filter_map(|r| r.ending_sequence_number) + .max(); + let start = next_sequence(&last_end.map_or(local.clone(), |end| end.max(local)))?; let starts: Vec = vec![start; SHARDS_PER_STREAM as usize]; engine .insert_stream_shards(account_ks, table_id, &label, view_type, &starts) @@ -295,6 +325,13 @@ impl CassandraEngine { /// Close every open shard of the table: its `EndingSequenceNumber` becomes /// the sequence number of its last record, or its starting number when it /// holds none (both measured 2026-10-24). + /// + /// A shard's range has no upper bound while it is open, so the ending + /// number is written only if the shard has none yet: a close that read the + /// shard as open and takes its ending after another close has written one + /// (a disable applying after an overlapping enable closed the stream and + /// started the next) would otherwise take a record of the next stream from + /// the shared partition. pub(crate) async fn close_stream_shards( &self, account_ks: &str, @@ -303,10 +340,26 @@ impl CassandraEngine { ) -> Result<(), StorageError> { let rows = self.stream_shard_rows(account_ks, Some(table_id)).await?; let ids = open_shard_ids_of(table_id, label, &rows); + let open = rows + .into_iter() + .filter(|r| ids.contains(&r.shard_id)) + .collect(); + self.close_shards_as_read(account_ks, table_id, open).await + } + + /// Write the ending numbers of `rows`, shard rows of table `table_id` this + /// close read as open: each one's last record after its start, or its + /// start when it holds none, unless the shard already has an ending + /// number. Public for the direct tests, which run a close that read its + /// rows before another close wrote them. + #[doc(hidden)] + pub async fn close_shards_as_read( + &self, + account_ks: &str, + table_id: &str, + rows: Vec, + ) -> Result<(), StorageError> { for row in rows { - if !ids.contains(&row.shard_id) { - continue; - } let range = ShardRange { physical: physical_shard_id(table_id, &row.shard_id), after: row.starting_sequence_number.clone(), @@ -320,7 +373,7 @@ impl CassandraEngine { &self.session, &format!( "UPDATE {account_ks}.stream_shards SET ending_sequence_number = ? \ - WHERE shard_id = ?" + WHERE shard_id = ? IF ending_sequence_number = null" ), cdrs_tokio::query_values!(ending.as_str(), row.shard_id.as_str()), "close_stream_shards", diff --git a/crates/storage-cassandra/tests/direct/stream_disable.rs b/crates/storage-cassandra/tests/direct/stream_disable.rs index daa87ba4a..4f59fa520 100644 --- a/crates/storage-cassandra/tests/direct/stream_disable.rs +++ b/crates/storage-cassandra/tests/direct/stream_disable.rs @@ -403,3 +403,232 @@ async fn overlapping_stream_changes_apply_exactly_one() { activate(&engine).await; } } + +/// The closed stream's last record came from a server whose clock is ahead of +/// this one: the new stream starts above that ending number, not at this +/// server's clock, so the record stays in one stream. +#[tokio::test] +async fn a_new_stream_starts_above_the_closed_streams_ending_numbers() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamSkew"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + let old_arn = describe_table(&engine, &account, name) + .await + .latest_stream_arn + .unwrap(); + set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let closed = describe_stream(&engine, &account, &old_arn).await; + // An ending number an hour ahead of this server's clock, as a record from + // a server that far ahead would leave it. + let ahead = format!( + "{:013}{:06}{:04}", + chrono::Utc::now().timestamp_millis() + 3_600_000, + 0, + 1 + ); + let account_ks = engine.account_keyspace(&account); + let update = format!( + "UPDATE {account_ks}.stream_shards SET ending_sequence_number = ? WHERE shard_id = ?" + ); + for shard in &closed.shards { + engine + .session_arc() + .query_with_values( + &update, + cdrs_tokio::query_values!(ahead.as_str(), shard.shard_id.as_str()), + ) + .await + .unwrap(); + } + + let reenabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new = describe_stream(&engine, &account, &reenabled.latest_stream_arn.unwrap()).await; + assert_eq!(new.shards.len(), 4); + for shard in &new.shards { + let start = &shard.sequence_number_range.starting_sequence_number; + assert_eq!(start.len(), ahead.len()); + assert!( + start.as_str() > ahead.as_str(), + "start {start} is not above the closed stream's ending number {ahead}" + ); + } +} + +/// A disable whose close read the old stream's shards as open and writes their +/// ending numbers only after an overlapping enable closed them, started the +/// new stream and the new stream took records: the old shards keep the ending +/// numbers the first close wrote, below the new stream's start. +#[tokio::test] +async fn a_late_close_keeps_the_ending_numbers_the_first_close_wrote() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamLateClose"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + let table = describe_table(&engine, &account, name).await; + let old_arn = table.latest_stream_arn.clone().unwrap(); + for i in 0..8 { + put( + &engine, + &account, + name, + &format!("a{i}"), + StreamViewType::KeysOnly, + ) + .await; + } + // The rows a disable reads before it closes: every shard open. + let read_open: Vec = + describe_stream(&engine, &account, &old_arn) + .await + .shards + .into_iter() + .map(|s| extenddb_storage::stream_lineage::ShardRow { + table_id: table.table_id.clone(), + shard_id: s.shard_id, + parent_shard_id: s.parent_shard_id, + starting_sequence_number: s.sequence_number_range.starting_sequence_number, + ending_sequence_number: None, + }) + .collect(); + assert_eq!(read_open.len(), 4); + + set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let closed = describe_stream(&engine, &account, &old_arn).await; + let reenabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new_arn = reenabled.latest_stream_arn.unwrap(); + for i in 0..8 { + put( + &engine, + &account, + name, + &format!("b{i}"), + StreamViewType::KeysOnly, + ) + .await; + } + let new = describe_stream(&engine, &account, &new_arn).await; + let mut new_records = 0; + for shard in &new.shards { + new_records += read_all(&engine, &account, &new_arn, &shard.shard_id) + .await + .len(); + } + assert_eq!(new_records, 8, "the new stream holds the eight new records"); + + // The late close of the old stream's label. + engine + .close_shards_as_read( + &engine.account_keyspace(&account), + &table.table_id, + read_open, + ) + .await + .unwrap(); + let after = describe_stream(&engine, &account, &old_arn).await; + for (a, b) in after.shards.iter().zip(&closed.shards) { + assert_eq!(a.shard_id, b.shard_id); + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number, + "the late close moved shard {}'s ending number", + a.shard_id + ); + } + let start = &new.shards[0].sequence_number_range.starting_sequence_number; + for shard in &after.shards { + let end = shard + .sequence_number_range + .ending_sequence_number + .as_deref() + .unwrap(); + assert!( + end < start.as_str(), + "ending {end} is not below the new start {start}" + ); + } +} diff --git a/docs/design/07-component-streams.md b/docs/design/07-component-streams.md index d87708787..168da7dbf 100755 --- a/docs/design/07-component-streams.md +++ b/docs/design/07-component-streams.md @@ -97,11 +97,11 @@ A table has a sequence of streams. `UpdateTable StreamEnabled=false` closes the Each stream's identity is carried in its shard ids, `shardId-{table_id}-{label_millis:013}{view_type:1}{index:02}`, so no backend stores a stream table. Shard ids from before this scheme (`shardId-{table_id}-{index}`, and the `shardId-{table_name}-{index}` form SQLite and PostgreSQL wrote before v0.1.14) belong to the table's current stream. Writes record only to open shards. The rules shared by the four backends live in `extenddb_storage::stream_lineage`; each backend reads its table row and shard rows and calls them. -On Cassandra the write paths compute a record's partition from the table id and partition key alone, inside the data write's LOGGED BATCH, so records stay in one partition per shard index and a stream's shard is a range of that partition: after its `StartingSequenceNumber`, up to its `EndingSequenceNumber` once closed. A new stream's shard starts where the closed one on the same index ended. +On Cassandra the write paths compute a record's partition from the table id and partition key alone, inside the data write's LOGGED BATCH, so records stay in one partition per shard index and a stream's shard is a range of that partition: after its `StartingSequenceNumber`, up to its `EndingSequenceNumber` once closed. A new stream's shards start one above the larger of the enabling server's next clock value and the last `EndingSequenceNumber` among the table's shards. A stream change travels alone: an `UpdateTable` that carries `StreamSpecification` beside `BillingMode` or `ProvisionedThroughput` is refused with `You cannot modify stream status while updating table IOPS`, beside any `GlobalSecondaryIndexUpdates` entry with `You cannot create or delete index while changing stream status`, and nothing is applied. A table created with `StreamSpecification {StreamEnabled: false}` stores nothing of it. -Writes racing a disable. The service's rule (measured 2026-10-24, 569 writes against one disable): every write accepted before the stream reports `DISABLED` lands in the closed stream before its shard's `EndingSequenceNumber`, and a write after that records nothing. ExtendDB closes the shards in the `UpdateTable` call. On SQLite the engine write lock serialises the close against every write. On PostgreSQL the write path reads the open shards `FOR SHARE` in its data transaction and the close locks them `FOR UPDATE` before taking the ending numbers, in one data transaction, so a write that read the open set commits before the ending number is taken and a later write re-evaluates the closed rows and records nothing. On Cassandra a new stream's shards start at a clock value drawn at enable time, above every sequence issued before, so a write of the closed stream that commits after the close falls between the old shard's ending number and the new shard's starting number and is read by neither stream (the sequence is drawn in the write's LOGGED BATCH, which the close cannot wait for). On MongoDB the window is one write transaction wide: a write whose transaction read the shard as open and commits after the close lands past the ending number; closing the gap needs the write transaction to touch the shard document and the stream path to classify the resulting write conflict as retryable, which it does not today. On every backend a write that found no open shard because its key info pre-dated the disable is acknowledged and recorded in no stream; the service records every write it acknowledges while the disable is in progress. A table toggled many times accumulates four closed shard rows per cycle; nothing sweeps a live table's closed shards. On PostgreSQL, MongoDB and Cassandra an enable commits its catalog row before inserting its shards, so a disable that reads the row in between closes whatever shards exist at that instant; the enable re-reads the row after its inserts and closes its own shards when the row no longer says enabled with its label, so a stream never reports `DISABLED` with open shards. +Writes racing a disable. The service's rule (measured 2026-10-24, 569 writes against one disable): every write accepted before the stream reports `DISABLED` lands in the closed stream before its shard's `EndingSequenceNumber`, and a write after that records nothing. ExtendDB closes the shards in the `UpdateTable` call. On SQLite the engine write lock serialises the close against every write. On PostgreSQL the write path reads the open shards without a lock, locks the one shard its record goes to `FOR KEY SHARE` while re-checking that it is still open, and the close locks the open rows `FOR UPDATE` before taking the ending numbers, in one data transaction, so a write that locked its row commits before the ending number is taken and a later write finds the row closed and records nothing. On Cassandra a new stream's shards start one above the larger of the enabling server's next clock value and the last `EndingSequenceNumber` among the table's shards, so on one server a write of the closed stream that commits after the close falls between the old shard's ending number and the new shard's starting number and is read by neither stream (the sequence is drawn in the write's LOGGED BATCH, which the close cannot wait for). A record of the old stream committed before the close by a server whose clock is ahead of the enabling server's is in the closed shard's ending number, so it is below the start and in one stream only; with several servers, a raced record from a server whose clock is ahead can land above the start, in the new stream. A server whose clock is behind the start can write a new-stream record below it, which no stream reads. Both cases are part of the deferred cross-instance consistency item below. On MongoDB the window is one write transaction wide: a write whose transaction read the shard as open and commits after the close lands past the ending number; closing the gap needs the write transaction to touch the shard document and the stream path to classify the resulting write conflict as retryable, which it does not today. On every backend a write that found no open shard because its key info pre-dated the disable is acknowledged and recorded in no stream; the service records every write it acknowledges while the disable is in progress. A table toggled many times accumulates four closed shard rows per cycle; nothing sweeps a live table's closed shards. On PostgreSQL, MongoDB and Cassandra an enable commits its catalog row before inserting its shards, so a disable that reads the row in between closes whatever shards exist at that instant; the enable re-reads the row after its inserts and closes its own shards when the row no longer says enabled with its label, so a stream never reports `DISABLED` with open shards. ## 5. Integration Points From c818c0a397e8c5507eb733c9fda74565d69064a1 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:32:22 +0000 Subject: [PATCH 10/15] docs(streams): upgrade order for stream changes, and what a disable stores The upgrade manual gets a behaviour change entry: a server on the earlier release writes records past a closed shard's ending number and re-enables a stream under its old ARN, so every server is upgraded before any UpdateTable changes a StreamSpecification, and rollback is a restore of the pre-upgrade database. No catalog version gate stops the earlier release, since the change needs no catalog migration. The entry also says an existing MongoDB deployment gets the new stream_shards index from extenddb migrate --yes. The PostgreSQL and MongoDB lifecycle comments and the design document said a disable clears the specification; it stores {StreamEnabled: false} with the stream's view type, which DescribeTable omits. DescribeBackup for a backup taken after a disable keeps showing StreamDescription as {StreamEnabled: false}, as before the view type was kept, since the service's answer there is not measured. --- crates/engine/src/backup.rs | 25 +++++++++++----- .../storage-mongodb/src/stream_lifecycle.rs | 6 ++-- .../storage-postgres/src/stream_lifecycle.rs | 6 ++-- docs/design/07-component-streams.md | 2 +- docs/manuals/07-upgrade-manual.md | 29 +++++++++++++++++++ 5 files changed, 54 insertions(+), 14 deletions(-) diff --git a/crates/engine/src/backup.rs b/crates/engine/src/backup.rs index dadeb5fbf..8d39ecdff 100755 --- a/crates/engine/src/backup.rs +++ b/crates/engine/src/backup.rs @@ -21,7 +21,7 @@ use extenddb_core::limits::LimitsConfig; use extenddb_core::types::{ BackupDescription, BackupSummary, BillingMode, CreateTableInput, GlobalSecondaryIndexInfo, LocalSecondaryIndexInfo, ProvisionedThroughput, RestoreSummary, SourceTableFeatureDetails, - SseDescription, TableDescription, TableStatus, + SseDescription, StreamSpecification, TableDescription, TableStatus, }; use extenddb_storage::BackupEngine; use extenddb_storage::error::StorageError; @@ -489,12 +489,23 @@ fn apply_source_to_description( }) .collect() }), - // A disabled specification is kept by the backends for the - // closed stream's view type; the service's DescribeBackup for - // a table whose stream was disabled is not measured, so the - // member is omitted as it was before the specification was - // kept. - stream_description: d.stream_specification.clone().filter(|s| s.stream_enabled), + // A backup of a table whose stream was disabled carries the + // stored `{StreamEnabled: false}`, and DescribeBackup shows it + // as `StreamDescription: {StreamEnabled: false}`, as it did + // before the view type was kept in the stored specification; + // the service's answer for that backup is not measured. The + // view type the backends now keep for the closed stream is + // dropped here so the answer stays the one before. + stream_description: d.stream_specification.clone().map(|s| { + if s.stream_enabled { + s + } else { + StreamSpecification { + stream_enabled: false, + stream_view_type: None, + } + } + }), time_to_live_description: d.time_to_live.clone(), sse_description: SseDescription::from_specification( d.sse_specification.as_ref(), diff --git a/crates/storage-mongodb/src/stream_lifecycle.rs b/crates/storage-mongodb/src/stream_lifecycle.rs index dfa658f64..da2d2a80c 100644 --- a/crates/storage-mongodb/src/stream_lifecycle.rs +++ b/crates/storage-mongodb/src/stream_lifecycle.rs @@ -194,9 +194,9 @@ impl MongoEngine { /// Decide an `UpdateTable` `StreamSpecification` against the table /// document and add its catalog fields to `update_doc`. /// - /// Disabling sets the specification to null, so DescribeTable omits - /// `StreamSpecification`, and keeps `stream_label`, so `LatestStreamArn` - /// keeps naming the disabled stream. Enabling stores the specification and + /// Disabling stores `{StreamEnabled: false}` with the stream's view type, + /// which DescribeTable omits, and keeps `stream_label`, so + /// `LatestStreamArn` keeps naming the disabled stream. Enabling stores the specification and /// a new label. The returned shard changes run once `update_doc` is /// written. This backend has no transaction around UpdateTable, so two /// concurrent stream changes can both pass the check. diff --git a/crates/storage-postgres/src/stream_lifecycle.rs b/crates/storage-postgres/src/stream_lifecycle.rs index 553fedaaf..b75967450 100644 --- a/crates/storage-postgres/src/stream_lifecycle.rs +++ b/crates/storage-postgres/src/stream_lifecycle.rs @@ -217,9 +217,9 @@ impl PostgresEngine { /// Apply an `UpdateTable` `StreamSpecification` in the caller's catalog /// transaction, which holds the table row `FOR UPDATE`. /// - /// Disabling clears the stored specification, so DescribeTable omits - /// `StreamSpecification`, and keeps `stream_label`, so `LatestStreamArn` - /// keeps naming the disabled stream. Enabling stores the specification and + /// Disabling stores `{StreamEnabled: false}` with the stream's view type, + /// which DescribeTable omits, and keeps `stream_label`, so + /// `LatestStreamArn` keeps naming the disabled stream. Enabling stores the specification and /// a new label. The shard changes are returned, for the caller to apply /// with [`PendingStream::apply`] once the catalog has committed: a request /// refused after this point (an invalid index update in the same call) diff --git a/docs/design/07-component-streams.md b/docs/design/07-component-streams.md index 168da7dbf..76250ee8f 100755 --- a/docs/design/07-component-streams.md +++ b/docs/design/07-component-streams.md @@ -93,7 +93,7 @@ DynamoDB retains stream records for 24 hours. Options: ### 4.5 Enable and Disable -A table has a sequence of streams. `UpdateTable StreamEnabled=false` closes the current one: the stored specification is cleared (DescribeTable omits `StreamSpecification`), `LatestStreamArn` and `LatestStreamLabel` keep naming the closed stream, DescribeStream reports it `DISABLED` with its original view type, and every open shard gets an `EndingSequenceNumber` (its last record's sequence number, or its starting number when it holds none). `StreamEnabled=true` afterwards creates a new stream: a new label later than the previous one, four new shard ids, the requested view type. The closed stream stays describable and listed; ListStreams lists every stream of a table, newest first. Enabling a table whose stream is enabled, and disabling a table with no enabled stream, are refused with `ValidationException`. +A table has a sequence of streams. `UpdateTable StreamEnabled=false` closes the current one: the stored specification becomes `{StreamEnabled: false}` with the stream's view type (DescribeTable omits a disabled `StreamSpecification`), `LatestStreamArn` and `LatestStreamLabel` keep naming the closed stream, DescribeStream reports it `DISABLED` with its original view type, and every open shard gets an `EndingSequenceNumber` (its last record's sequence number, or its starting number when it holds none). `StreamEnabled=true` afterwards creates a new stream: a new label later than the previous one, four new shard ids, the requested view type. The closed stream stays describable and listed; ListStreams lists every stream of a table, newest first. Enabling a table whose stream is enabled, and disabling a table with no enabled stream, are refused with `ValidationException`. Each stream's identity is carried in its shard ids, `shardId-{table_id}-{label_millis:013}{view_type:1}{index:02}`, so no backend stores a stream table. Shard ids from before this scheme (`shardId-{table_id}-{index}`, and the `shardId-{table_name}-{index}` form SQLite and PostgreSQL wrote before v0.1.14) belong to the table's current stream. Writes record only to open shards. The rules shared by the four backends live in `extenddb_storage::stream_lineage`; each backend reads its table row and shard rows and calls them. diff --git a/docs/manuals/07-upgrade-manual.md b/docs/manuals/07-upgrade-manual.md index f4451a0f5..ac35e86d1 100755 --- a/docs/manuals/07-upgrade-manual.md +++ b/docs/manuals/07-upgrade-manual.md @@ -319,6 +319,35 @@ Catalog upgrades change the schema; behavior changes alter how the running serve interprets existing data or configuration. Review these before upgrading a live deployment, even when no catalog migration is required. +### Streams: disabling closes the shards, enabling creates a new stream + +**What changed.** `UpdateTable` with `StreamEnabled: false` now closes the +stream's shards (each gets an `EndingSequenceNumber`), and `StreamEnabled: true` +afterwards creates a new stream with a new ARN and four new shard ids, as on +DynamoDB. Before this release a disable left the shards open and a re-enable +kept the same ARN. Each stream's identity is carried in its shard ids, so no +catalog migration is needed and no catalog version gate stops an earlier release +from serving the same database. + +**Upgrade every server before changing a table's stream.** A server still on +the earlier release does not know closed shards: it writes records past a +shard's `EndingSequenceNumber`, where no reader finds them, and re-enables a +disabled stream under its old ARN instead of creating a new one. In a +deployment with several servers, finish upgrading all of them before any +`UpdateTable` changes a `StreamSpecification`. A stream disabled by the earlier +release keeps its four shards open until the first `UpdateTable` on that table +under the new release (an enable, or a disable, which is refused as before and +closes them). + +**MongoDB.** An existing deployment gets the new `{table_id: 1, shard_id: 1}` +index on `stream_shards` from `extenddb migrate --yes`; `extenddb init` creates +it on a new one. + +**Rollback.** Going back to the earlier release means restoring the +pre-upgrade database as in [Rollback Procedure](#rollback-procedure); the +earlier release does not read the closed shards' ending numbers or the labelled +shard ids the new release writes. + ### v0.1.14 LeadingKeys now enforced on Query, batch and transaction operations **What changed.** `dynamodb:LeadingKeys` is now evaluated on `Query`, From 5433ed3e648f8f69014dc2846715c0d6215242b0 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:36:21 +0000 Subject: [PATCH 11/15] refactor(streams): move the engine stream tests and the ListStreams builder to their own files Pure moves, no behaviour change: crates/engine/src/streams.rs had grown to 711 lines with the GetShardIterator bounds, so its unit tests move to streams_tests.rs; crates/storage/src/stream_lineage.rs had reached 602, so the ListStreams page builder moves to stream_lineage_list.rs and is re-exported from the parent module. --- crates/engine/src/streams.rs | 217 +--------------------- crates/engine/src/streams_tests.rs | 217 ++++++++++++++++++++++ crates/storage/src/stream_lineage.rs | 63 +------ crates/storage/src/stream_lineage_list.rs | 68 +++++++ 4 files changed, 291 insertions(+), 274 deletions(-) create mode 100644 crates/engine/src/streams_tests.rs create mode 100644 crates/storage/src/stream_lineage_list.rs diff --git a/crates/engine/src/streams.rs b/crates/engine/src/streams.rs index 6b457cbbd..c88127ac2 100755 --- a/crates/engine/src/streams.rs +++ b/crates/engine/src/streams.rs @@ -494,218 +494,5 @@ fn storage_to_dynamo(e: StorageError) -> DynamoDbError { } #[cfg(test)] -mod tests { - /// AT_SEQUENCE_NUMBER converts to AFTER by subtracting 1 and padding to - /// the backend's stored width. Verify that an unpadded client input is - /// normalised correctly for both the 21-digit (postgres/sqlite/mongodb) - /// and 23-digit (cassandra) cases, and that the edge case n=0 maps to - /// TRIM_HORIZON (empty string). - #[test] - fn at_sequence_number_padding() { - let cases: &[(&str, usize, &str)] = &[ - ("5", 21, "000000000000000000004"), - ("000000000000000000005", 21, "000000000000000000004"), - ("5", 23, "00000000000000000000004"), - ("00000000000000000000005", 23, "00000000000000000000004"), - ("0", 21, ""), - ("1", 21, "000000000000000000000"), - ]; - for (input, width, expected) in cases { - let n: u128 = input.parse().unwrap(); - let result = if n > 0 { - format!("{:0>width$}", n - 1) - } else { - String::new() - }; - assert_eq!(&result, expected, "input={input} width={width}"); - } - } - - #[test] - fn shard_iterator_round_trip_backward_compat_and_fail_closed() { - let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; - let enc = - super::encode_shard_iterator("shardId-1", "AFTER_SEQUENCE_NUMBER", "42", 1000, stream); - let d = super::decode_shard_iterator(&enc).expect("decode"); - assert_eq!(d.shard_id, "shardId-1"); - assert_eq!(d.sequence, "42"); - assert_eq!(d.created_at, Some(1000)); - assert_eq!(d.stream_arn.as_deref(), Some(stream)); - - // A token from an older build (four fields, no stream ARN) still - // decodes, with stream_arn None, so the authorization layer can fail - // closed on it. - let old = base64::Engine::encode( - &base64::engine::general_purpose::STANDARD, - "shardId-2|AFTER_SEQUENCE_NUMBER|7|1000", - ); - let d2 = super::decode_shard_iterator(&old).expect("decode old"); - assert_eq!(d2.shard_id, "shardId-2"); - assert_eq!(d2.created_at, Some(1000)); - assert_eq!(d2.stream_arn, None); - - // Garbage does not decode. - assert!(super::decode_shard_iterator("!!!not base64!!!").is_none()); - } - - #[test] - fn stream_not_found_maps_to_resource_not_found_with_the_arn() { - // `describe_stream` answers a stream ARN that names no stream with the - // storage not-found error that carries the ARN. - let arn = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; - match super::storage_to_dynamo(extenddb_storage::util::stream_not_found(arn)) { - super::DynamoDbError::ResourceNotFoundException(msg) => assert_eq!( - msg, - format!("Requested resource not found: Stream: {arn} not found") - ), - other => panic!("expected ResourceNotFoundException, got {other:?}"), - } - } - - #[test] - fn stream_does_not_exist_maps_to_resource_not_found() { - // The answer for a stream ARN that names no stream, and for a shard - // that is not one of the stream's (`get_stream_records`). - match super::storage_to_dynamo(extenddb_storage::util::stream_does_not_exist()) { - super::DynamoDbError::ResourceNotFoundException(msg) => { - assert_eq!(msg, "Requested resource not found: Stream does not exist"); - } - other => panic!("expected ResourceNotFoundException, got {other:?}"), - } - } - - #[test] - fn get_records_limit_bounds_carry_the_measured_sentences() { - let refused = |l: i64| match super::get_records_limit(Some(l)) { - Err(super::DynamoDbError::ValidationException(msg)) => msg, - other => panic!("expected ValidationException for Limit {l}, got {other:?}"), - }; - for l in [0, -1] { - assert_eq!( - refused(l), - "1 validation error detected: Value at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" - ); - } - assert_eq!( - refused(1001), - "One or more parameter values were invalid: Limit should be less than or equal to 1000" - ); - assert_eq!(super::get_records_limit(Some(1)).unwrap(), 1); - assert_eq!(super::get_records_limit(Some(1000)).unwrap(), 1000); - assert_eq!(super::get_records_limit(None).unwrap(), 1000); - } - - fn b64(s: &str) -> String { - base64::Engine::encode(&base64::engine::general_purpose::STANDARD, s) - } - - #[test] - fn the_shard_id_maximum_applies_once_the_shard_is_not_found() { - // A shard of a table named with 41 characters, as a release before - // v0.1.14 wrote it: 8 + 41 + 1 + 16 = 66 characters. Found, it is not - // this function's case at all; not found, the model's sentence. - let long = format!("shardId-{}-{:016}", "t".repeat(41), 3); - assert_eq!(long.chars().count(), 66); - let sentence = "1 validation error detected: Value at 'shardId' failed to satisfy constraint: Member must have length less than or equal to 65"; - for err in [ - extenddb_storage::util::shard_does_not_exist(), - extenddb_storage::util::stream_does_not_exist(), - ] { - match super::shard_lookup_error(long.chars().count(), err) { - super::DynamoDbError::ValidationException(msg) => assert_eq!(msg, sentence), - other => panic!("expected ValidationException, got {other:?}"), - } - } - // Within the bound, the lookup's own answer stands. - match super::shard_lookup_error(65, extenddb_storage::util::shard_does_not_exist()) { - super::DynamoDbError::ResourceNotFoundException(msg) => { - assert_eq!(msg, extenddb_storage::util::SHARD_DOES_NOT_EXIST); - } - other => panic!("expected ResourceNotFoundException, got {other:?}"), - } - } - - #[test] - fn the_constraint_sentences_match_the_measured_ones() { - assert_eq!( - super::constraint("shardId", "have length greater than or equal to 28"), - super::SHARD_ID_TOO_SHORT - ); - assert_eq!( - super::constraint("limit", "have value greater than or equal to 1"), - "1 validation error detected: Value at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" - ); - } - - #[test] - fn classify_current_token_is_stream() { - let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; - let enc = - super::encode_shard_iterator("shardId-1", "AFTER_SEQUENCE_NUMBER", "42", 1000, stream); - match super::classify_shard_iterator(&enc) { - super::ShardIteratorClass::Stream(arn) => assert_eq!(arn, stream), - other => panic!("expected Stream, got {other:?}"), - } - } - - #[test] - fn classify_old_format_four_fields_is_expired() { - // Four pre-upgrade fields with a parseable timestamp, no stream ARN. - let old = b64("shardId-2|AFTER_SEQUENCE_NUMBER|7|1000"); - assert!(matches!( - super::classify_shard_iterator(&old), - super::ShardIteratorClass::Expired - )); - } - - #[test] - fn classify_undecodable_or_short_or_unparseable_is_invalid() { - // Not base64. - assert!(matches!( - super::classify_shard_iterator("!!!not base64!!!"), - super::ShardIteratorClass::Invalid - )); - // Three fields: decodes but is not the old four-field shape. - let three = b64("shardId-3|AFTER_SEQUENCE_NUMBER|7"); - assert!(matches!( - super::classify_shard_iterator(&three), - super::ShardIteratorClass::Invalid - )); - // Four fields but an unparseable timestamp. - let bad_ts = b64("shardId-4|AFTER_SEQUENCE_NUMBER|7|notanumber"); - assert!(matches!( - super::classify_shard_iterator(&bad_ts), - super::ShardIteratorClass::Invalid - )); - } - - #[test] - fn a_token_without_a_stream_arn_gets_the_answer_the_authorization_layer_gives() { - // Two and three fields have no timestamp (invalid), four fields with a - // timestamp come from before the ARN was carried (expired), and a - // timestamp that is not a number is invalid. - for token in [ - b64("shardId-5|AFTER_SEQUENCE_NUMBER"), - b64("shardId-3|AFTER_SEQUENCE_NUMBER|7"), - b64("shardId-2|AFTER_SEQUENCE_NUMBER|7|1000"), - b64("shardId-4|AFTER_SEQUENCE_NUMBER|7|notanumber"), - ] { - let it = super::decode_shard_iterator(&token).expect("the token decodes"); - let expected = match super::classify_shard_iterator(&token) { - super::ShardIteratorClass::Expired => { - super::DynamoDbError::ExpiredIteratorException( - super::SHARD_ITERATOR_EXPIRED_MESSAGE.to_owned(), - ) - } - super::ShardIteratorClass::Invalid => { - super::DynamoDbError::ValidationException("Invalid ShardIterator".to_owned()) - } - super::ShardIteratorClass::Stream(_) => panic!("the token carries no stream ARN"), - }; - assert_eq!( - format!("{:?}", super::no_stream_arn_error(&it)), - format!("{expected:?}") - ); - } - } -} +#[path = "streams_tests.rs"] +mod tests; diff --git a/crates/engine/src/streams_tests.rs b/crates/engine/src/streams_tests.rs new file mode 100644 index 000000000..1504421fd --- /dev/null +++ b/crates/engine/src/streams_tests.rs @@ -0,0 +1,217 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Unit tests for the streams request handlers: iterator tokens, the measured +//! constraint sentences and the not-found answers. + +/// AT_SEQUENCE_NUMBER converts to AFTER by subtracting 1 and padding to +/// the backend's stored width. Verify that an unpadded client input is +/// normalised correctly for both the 21-digit (postgres/sqlite/mongodb) +/// and 23-digit (cassandra) cases, and that the edge case n=0 maps to +/// TRIM_HORIZON (empty string). +#[test] +fn at_sequence_number_padding() { + let cases: &[(&str, usize, &str)] = &[ + ("5", 21, "000000000000000000004"), + ("000000000000000000005", 21, "000000000000000000004"), + ("5", 23, "00000000000000000000004"), + ("00000000000000000000005", 23, "00000000000000000000004"), + ("0", 21, ""), + ("1", 21, "000000000000000000000"), + ]; + for (input, width, expected) in cases { + let n: u128 = input.parse().unwrap(); + let result = if n > 0 { + format!("{:0>width$}", n - 1) + } else { + String::new() + }; + assert_eq!(&result, expected, "input={input} width={width}"); + } +} + +#[test] +fn shard_iterator_round_trip_backward_compat_and_fail_closed() { + let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; + let enc = + super::encode_shard_iterator("shardId-1", "AFTER_SEQUENCE_NUMBER", "42", 1000, stream); + let d = super::decode_shard_iterator(&enc).expect("decode"); + assert_eq!(d.shard_id, "shardId-1"); + assert_eq!(d.sequence, "42"); + assert_eq!(d.created_at, Some(1000)); + assert_eq!(d.stream_arn.as_deref(), Some(stream)); + + // A token from an older build (four fields, no stream ARN) still + // decodes, with stream_arn None, so the authorization layer can fail + // closed on it. + let old = base64::Engine::encode( + &base64::engine::general_purpose::STANDARD, + "shardId-2|AFTER_SEQUENCE_NUMBER|7|1000", + ); + let d2 = super::decode_shard_iterator(&old).expect("decode old"); + assert_eq!(d2.shard_id, "shardId-2"); + assert_eq!(d2.created_at, Some(1000)); + assert_eq!(d2.stream_arn, None); + + // Garbage does not decode. + assert!(super::decode_shard_iterator("!!!not base64!!!").is_none()); +} + +#[test] +fn stream_not_found_maps_to_resource_not_found_with_the_arn() { + // `describe_stream` answers a stream ARN that names no stream with the + // storage not-found error that carries the ARN. + let arn = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; + match super::storage_to_dynamo(extenddb_storage::util::stream_not_found(arn)) { + super::DynamoDbError::ResourceNotFoundException(msg) => assert_eq!( + msg, + format!("Requested resource not found: Stream: {arn} not found") + ), + other => panic!("expected ResourceNotFoundException, got {other:?}"), + } +} + +#[test] +fn stream_does_not_exist_maps_to_resource_not_found() { + // The answer for a stream ARN that names no stream, and for a shard + // that is not one of the stream's (`get_stream_records`). + match super::storage_to_dynamo(extenddb_storage::util::stream_does_not_exist()) { + super::DynamoDbError::ResourceNotFoundException(msg) => { + assert_eq!(msg, "Requested resource not found: Stream does not exist"); + } + other => panic!("expected ResourceNotFoundException, got {other:?}"), + } +} + +#[test] +fn get_records_limit_bounds_carry_the_measured_sentences() { + let refused = |l: i64| match super::get_records_limit(Some(l)) { + Err(super::DynamoDbError::ValidationException(msg)) => msg, + other => panic!("expected ValidationException for Limit {l}, got {other:?}"), + }; + for l in [0, -1] { + assert_eq!( + refused(l), + "1 validation error detected: Value at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ); + } + assert_eq!( + refused(1001), + "One or more parameter values were invalid: Limit should be less than or equal to 1000" + ); + assert_eq!(super::get_records_limit(Some(1)).unwrap(), 1); + assert_eq!(super::get_records_limit(Some(1000)).unwrap(), 1000); + assert_eq!(super::get_records_limit(None).unwrap(), 1000); +} + +fn b64(s: &str) -> String { + base64::Engine::encode(&base64::engine::general_purpose::STANDARD, s) +} + +#[test] +fn the_shard_id_maximum_applies_once_the_shard_is_not_found() { + // A shard of a table named with 41 characters, as a release before + // v0.1.14 wrote it: 8 + 41 + 1 + 16 = 66 characters. Found, it is not + // this function's case at all; not found, the model's sentence. + let long = format!("shardId-{}-{:016}", "t".repeat(41), 3); + assert_eq!(long.chars().count(), 66); + let sentence = "1 validation error detected: Value at 'shardId' failed to satisfy constraint: Member must have length less than or equal to 65"; + for err in [ + extenddb_storage::util::shard_does_not_exist(), + extenddb_storage::util::stream_does_not_exist(), + ] { + match super::shard_lookup_error(long.chars().count(), err) { + super::DynamoDbError::ValidationException(msg) => assert_eq!(msg, sentence), + other => panic!("expected ValidationException, got {other:?}"), + } + } + // Within the bound, the lookup's own answer stands. + match super::shard_lookup_error(65, extenddb_storage::util::shard_does_not_exist()) { + super::DynamoDbError::ResourceNotFoundException(msg) => { + assert_eq!(msg, extenddb_storage::util::SHARD_DOES_NOT_EXIST); + } + other => panic!("expected ResourceNotFoundException, got {other:?}"), + } +} + +#[test] +fn the_constraint_sentences_match_the_measured_ones() { + assert_eq!( + super::constraint("shardId", "have length greater than or equal to 28"), + super::SHARD_ID_TOO_SHORT + ); + assert_eq!( + super::constraint("limit", "have value greater than or equal to 1"), + "1 validation error detected: Value at 'limit' failed to satisfy constraint: Member must have value greater than or equal to 1" + ); +} + +#[test] +fn classify_current_token_is_stream() { + let stream = "arn:aws:dynamodb:us-east-1:1:table/T/stream/2026-10-20T16:17:40.188"; + let enc = + super::encode_shard_iterator("shardId-1", "AFTER_SEQUENCE_NUMBER", "42", 1000, stream); + match super::classify_shard_iterator(&enc) { + super::ShardIteratorClass::Stream(arn) => assert_eq!(arn, stream), + other => panic!("expected Stream, got {other:?}"), + } +} + +#[test] +fn classify_old_format_four_fields_is_expired() { + // Four pre-upgrade fields with a parseable timestamp, no stream ARN. + let old = b64("shardId-2|AFTER_SEQUENCE_NUMBER|7|1000"); + assert!(matches!( + super::classify_shard_iterator(&old), + super::ShardIteratorClass::Expired + )); +} + +#[test] +fn classify_undecodable_or_short_or_unparseable_is_invalid() { + // Not base64. + assert!(matches!( + super::classify_shard_iterator("!!!not base64!!!"), + super::ShardIteratorClass::Invalid + )); + // Three fields: decodes but is not the old four-field shape. + let three = b64("shardId-3|AFTER_SEQUENCE_NUMBER|7"); + assert!(matches!( + super::classify_shard_iterator(&three), + super::ShardIteratorClass::Invalid + )); + // Four fields but an unparseable timestamp. + let bad_ts = b64("shardId-4|AFTER_SEQUENCE_NUMBER|7|notanumber"); + assert!(matches!( + super::classify_shard_iterator(&bad_ts), + super::ShardIteratorClass::Invalid + )); +} + +#[test] +fn a_token_without_a_stream_arn_gets_the_answer_the_authorization_layer_gives() { + // Two and three fields have no timestamp (invalid), four fields with a + // timestamp come from before the ARN was carried (expired), and a + // timestamp that is not a number is invalid. + for token in [ + b64("shardId-5|AFTER_SEQUENCE_NUMBER"), + b64("shardId-3|AFTER_SEQUENCE_NUMBER|7"), + b64("shardId-2|AFTER_SEQUENCE_NUMBER|7|1000"), + b64("shardId-4|AFTER_SEQUENCE_NUMBER|7|notanumber"), + ] { + let it = super::decode_shard_iterator(&token).expect("the token decodes"); + let expected = match super::classify_shard_iterator(&token) { + super::ShardIteratorClass::Expired => super::DynamoDbError::ExpiredIteratorException( + super::SHARD_ITERATOR_EXPIRED_MESSAGE.to_owned(), + ), + super::ShardIteratorClass::Invalid => { + super::DynamoDbError::ValidationException("Invalid ShardIterator".to_owned()) + } + super::ShardIteratorClass::Stream(_) => panic!("the token carries no stream ARN"), + }; + assert_eq!( + format!("{:?}", super::no_stream_arn_error(&it)), + format!("{expected:?}") + ); + } +} diff --git a/crates/storage/src/stream_lineage.rs b/crates/storage/src/stream_lineage.rs index c40c9b56c..877e0ab53 100644 --- a/crates/storage/src/stream_lineage.rs +++ b/crates/storage/src/stream_lineage.rs @@ -23,7 +23,7 @@ use extenddb_core::types::{ KeySchemaElement, SequenceNumberRange, Shard, StreamDescription, StreamSpecification, - StreamStatus, StreamSummary, StreamViewType, + StreamStatus, StreamViewType, }; use crate::error::StorageError; @@ -574,64 +574,9 @@ pub fn invalid_exclusive_start_stream_arn(arn: &str) -> StorageError { )) } -/// Build a `ListStreams` page from the account's tables and their shards. -/// -/// Every stream of every table is listed, disabled ones included, in the -/// order the service answered on 2026-10-24: table names descending, and a -/// table's streams newest first. `exclusive_start` is the (table name, label) -/// of the previous page's last stream; `arn` formats a stream ARN. -/// `LastEvaluatedStreamArn` is present whenever `limit` cut the page, even -/// when no stream follows: with two streams and `Limit=1`, page two carries it -/// and page three is empty (measured 2026-10-24). -/// -/// # Errors -/// -/// `Validation` when `exclusive_start` names a table other than `table_name`. -pub fn list_streams( - tables: &[TableStreamRow], - shards: &[ShardRow], - table_name: Option<&str>, - limit: i64, - exclusive_start: Option<(String, String)>, - arn: impl Fn(&str, &str) -> String, -) -> Result<(Vec, Option), StorageError> { - if let (Some(name), Some((start_table, start_label))) = (table_name, &exclusive_start) - && start_table != name - { - return Err(invalid_exclusive_start_stream_arn(&arn( - start_table, - start_label, - ))); - } - let mut listed: Vec<(String, String)> = tables - .iter() - .filter(|t| table_name.is_none_or(|n| t.table_name == n)) - .flat_map(|t| { - table_streams(t, shards) - .into_iter() - .map(|s| (t.table_name.clone(), s.label)) - }) - .collect(); - listed.sort_by(|a, b| b.cmp(a)); - if let Some(start) = exclusive_start { - listed.retain(|entry| *entry < start); - } - let limit = usize::try_from(limit).unwrap_or(0).max(1); - let last = (listed.len() >= limit).then(|| { - let (t, l) = &listed[limit - 1]; - arn(t, l) - }); - listed.truncate(limit); - let summaries = listed - .into_iter() - .map(|(table_name, label)| StreamSummary { - stream_arn: arn(&table_name, &label), - stream_label: label, - table_name, - }) - .collect(); - Ok((summaries, last)) -} +#[path = "stream_lineage_list.rs"] +mod list; +pub use list::list_streams; #[cfg(test)] #[path = "stream_lineage_tests.rs"] diff --git a/crates/storage/src/stream_lineage_list.rs b/crates/storage/src/stream_lineage_list.rs new file mode 100644 index 000000000..a25d813b8 --- /dev/null +++ b/crates/storage/src/stream_lineage_list.rs @@ -0,0 +1,68 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The `ListStreams` page over a table's streams; see the parent module. + +use extenddb_core::types::StreamSummary; + +use super::{ShardRow, TableStreamRow, invalid_exclusive_start_stream_arn, table_streams}; +use crate::error::StorageError; + +/// Build a `ListStreams` page from the account's tables and their shards. +/// +/// Every stream of every table is listed, disabled ones included, in the +/// order the service answered on 2026-10-24: table names descending, and a +/// table's streams newest first. `exclusive_start` is the (table name, label) +/// of the previous page's last stream; `arn` formats a stream ARN. +/// `LastEvaluatedStreamArn` is present whenever `limit` cut the page, even +/// when no stream follows: with two streams and `Limit=1`, page two carries it +/// and page three is empty (measured 2026-10-24). +/// +/// # Errors +/// +/// `Validation` when `exclusive_start` names a table other than `table_name`. +pub fn list_streams( + tables: &[TableStreamRow], + shards: &[ShardRow], + table_name: Option<&str>, + limit: i64, + exclusive_start: Option<(String, String)>, + arn: impl Fn(&str, &str) -> String, +) -> Result<(Vec, Option), StorageError> { + if let (Some(name), Some((start_table, start_label))) = (table_name, &exclusive_start) + && start_table != name + { + return Err(invalid_exclusive_start_stream_arn(&arn( + start_table, + start_label, + ))); + } + let mut listed: Vec<(String, String)> = tables + .iter() + .filter(|t| table_name.is_none_or(|n| t.table_name == n)) + .flat_map(|t| { + table_streams(t, shards) + .into_iter() + .map(|s| (t.table_name.clone(), s.label)) + }) + .collect(); + listed.sort_by(|a, b| b.cmp(a)); + if let Some(start) = exclusive_start { + listed.retain(|entry| *entry < start); + } + let limit = usize::try_from(limit).unwrap_or(0).max(1); + let last = (listed.len() >= limit).then(|| { + let (t, l) = &listed[limit - 1]; + arn(t, l) + }); + listed.truncate(limit); + let summaries = listed + .into_iter() + .map(|(table_name, label)| StreamSummary { + stream_arn: arn(&table_name, &label), + stream_label: label, + table_name, + }) + .collect(); + Ok((summaries, last)) +} From 53e42fb640238817274f2470a90a7ad0299359cc Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 11:58:52 +0000 Subject: [PATCH 12/15] test(postgres): pin stream labels through the shard ids the stream carries The GetRecords ownership and page-order tests pinned a table's stream label by updating the catalog row alone. A shard id now carries its stream's label, so the ownership fixture's relabel moves each shard row and its records to the ids the new label gives them, and the page-order test reads under the label the table was created with. The relabel test also checks that the old shard id under the new ARN is refused. --- .../tests/stream_records_page_order.rs | 15 ++-- .../tests/stream_shard_ownership.rs | 79 +++++++++++++++---- 2 files changed, 71 insertions(+), 23 deletions(-) diff --git a/crates/storage-postgres/tests/stream_records_page_order.rs b/crates/storage-postgres/tests/stream_records_page_order.rs index 11e55be1a..9d1bfad24 100644 --- a/crates/storage-postgres/tests/stream_records_page_order.rs +++ b/crates/storage-postgres/tests/stream_records_page_order.rs @@ -23,7 +23,6 @@ use sqlx::postgres::PgPoolOptions; const ACCOUNT: &str = "111111111111"; const REGION: &str = "us-east-1"; -const LABEL: &str = "2026-01-01T00:00:00.000"; fn base_conn() -> Option { std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING") @@ -144,12 +143,12 @@ async fn a_page_is_in_sequence_order_when_the_records_were_written_in_reverse() .await .expect("create a stream-enabled table"); let table_id = desc.table_id.clone(); - sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_id = $2") - .bind(LABEL) - .bind(&table_id) - .execute(&db) - .await - .expect("pin the stream label"); + // The stream's label is carried in its shard ids, so the ARN takes the + // label the table was created with rather than pinning one. + let label = desc + .latest_stream_label + .clone() + .expect("a stream-enabled table has a label"); let shard: String = sqlx::query_scalar( "SELECT shard_id FROM stream_shards WHERE table_id = $1 ORDER BY shard_id LIMIT 1", ) @@ -178,7 +177,7 @@ async fn a_page_is_in_sequence_order_when_the_records_were_written_in_reverse() .await .expect("analyze the records"); - let arn = stream_arn(REGION, ACCOUNT, "t", LABEL); + let arn = stream_arn(REGION, ACCOUNT, "t", &label); let want = |from: u64| -> Vec { (from..from + 3).map(|n| format!("{n:021}")).collect() }; diff --git a/crates/storage-postgres/tests/stream_shard_ownership.rs b/crates/storage-postgres/tests/stream_shard_ownership.rs index 8c5a1cc90..cfcb4719b 100644 --- a/crates/storage-postgres/tests/stream_shard_ownership.rs +++ b/crates/storage-postgres/tests/stream_shard_ownership.rs @@ -24,6 +24,7 @@ use extenddb_core::types::{ ScalarAttributeType, StreamSpecification, StreamViewType, }; use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ShardStream, parse_shard_id, shard_id}; use extenddb_storage::util::stream_arn; use extenddb_storage::{StreamEngine, StreamRecordsResult, TableEngine}; use extenddb_storage_postgres::stream_records_sql::{ @@ -73,20 +74,19 @@ impl Scratch { .create_table(account, stream_table(name)) .await .expect("create a stream-enabled table"); - sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_id = $2") - .bind(label) - .bind(&desc.table_id) - .execute(&self.db) - .await - .expect("pin the stream label"); - let shard: String = sqlx::query_scalar( + self.relabel(&desc.table_id, label).await; + let shard = self.first_shard(&desc.table_id).await; + (desc.table_id, shard) + } + + async fn first_shard(&self, table_id: &str) -> String { + sqlx::query_scalar( "SELECT shard_id FROM stream_shards WHERE table_id = $1 ORDER BY shard_id LIMIT 1", ) - .bind(&desc.table_id) + .bind(table_id) .fetch_one(&self.db) .await - .expect("one shard id"); - (desc.table_id, shard) + .expect("one shard id") } async fn write_records(&self, shard_id: &str, table_id: &str, seqs: &[u64]) { @@ -130,7 +130,48 @@ impl Scratch { .expect("analyze the records"); } + /// Pin the table's stream label to `label`. A shard id carries its + /// stream's label, so each shard row of the table, and the records keyed + /// on it, move to the id the stream `label` gives its index. async fn relabel(&self, table_id: &str, label: &str) { + let ids: Vec = + sqlx::query_scalar("SELECT shard_id FROM stream_shards WHERE table_id = $1") + .bind(table_id) + .fetch_all(&self.db) + .await + .expect("shard ids"); + for old in ids { + let Some((ShardStream::Labelled { view_type, .. }, index)) = + parse_shard_id(table_id, &old) + else { + panic!("{old} is not a labelled shard of {table_id}"); + }; + let new = shard_id(table_id, label, view_type, index).expect("label shape"); + // The records reference the shard row, so the new row comes first + // and the old one goes last. + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, parent_shard_id, \ + starting_sequence_number, ending_sequence_number) \ + SELECT $1, table_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number FROM stream_shards WHERE shard_id = $2", + ) + .bind(&new) + .bind(&old) + .execute(&self.db) + .await + .expect("insert the relabelled shard row"); + sqlx::query("UPDATE stream_records SET shard_id = $1 WHERE shard_id = $2") + .bind(&new) + .bind(&old) + .execute(&self.db) + .await + .expect("move the shard's records"); + sqlx::query("DELETE FROM stream_shards WHERE shard_id = $1") + .bind(&old) + .execute(&self.db) + .await + .expect("remove the old shard row"); + } sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_id = $2") .bind(label) .bind(table_id) @@ -404,8 +445,9 @@ async fn another_accounts_same_named_stream_is_not_found() { s.cleanup().await; } -/// The table takes a new stream label while keeping its shard rows: the old -/// ARN's iterator is refused and the new ARN reads every record. +/// The table's stream takes a new label while keeping its records: the old +/// ARN's iterator, and the old shard id under the new ARN, are refused, and +/// the new stream's shard reads every record. #[tokio::test] async fn an_earlier_label_on_the_same_shards_is_not_found() { if base_conn().is_none() { @@ -422,16 +464,23 @@ async fn an_earlier_label_on_the_same_shards_is_not_found() { .expect("current label reads"); s.relabel(&table_id, LATER_LABEL).await; - s.write_records(&shard, &table_id, &[2]).await; + let new_shard = s.first_shard(&table_id).await; + assert_ne!(new_shard, shard); + s.write_records(&new_shard, &table_id, &[2]).await; + let new_arn = stream_arn(REGION, ACCOUNT, "t", LATER_LABEL); assert_stream_not_found( s.engine .get_stream_records(ACCOUNT, &old_arn, &shard, None, 10) .await, ); - let new_arn = stream_arn(REGION, ACCOUNT, "t", LATER_LABEL); + assert_stream_not_found( + s.engine + .get_stream_records(ACCOUNT, &new_arn, &shard, None, 10) + .await, + ); let (records, _) = s .engine - .get_stream_records(ACCOUNT, &new_arn, &shard, None, 10) + .get_stream_records(ACCOUNT, &new_arn, &new_shard, None, 10) .await .expect("new label reads"); assert_eq!(records.len(), 2); From a8d80ef3023f856b8a1b9d3cfe32a2f06d3cd042 Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:22:47 +0000 Subject: [PATCH 13/15] fix(streams): an enable closes the open shards of every earlier stream On PostgreSQL, MongoDB and Cassandra a stream change writes the catalog row first and changes the shards after it. An enable closed only the open shards of the label it read. When a disable's close failed after its catalog write and the client then enabled the stream, and that enable's close failed too, the table was enabled under the new label with no shards of its own while the earlier stream's four shards stayed open: writes recorded into a DISABLED stream, a later disable closed only the new label's rows, and every later enable closed only the label it read, so those shards were never closed. An enable now closes the open shards of every stream of the table older than the one it creates, legacy rows included (stream_lineage::open_shard_ids_before). A disable still closes only the stream it read, so a disable that applies after an overlapping enable leaves the new stream open. SQLite closes every open shard of the table in the transaction that writes the catalog row, so it is unchanged. PostgreSQL storage tests: an_enable_closes_an_earlier_stream_left_open seeds an enabled label with no shards beside an earlier stream's open shards, and the next disable and enable close them; a_disable_applying_after_an_overlapping_enable_leaves_the_new_stream_open applies a disable's shard changes for the earlier label after the enable created the new stream, and the new stream's four shards stay open. --- .../storage-cassandra/src/stream_lifecycle.rs | 58 ++++++-- .../storage-mongodb/src/stream_lifecycle.rs | 65 ++++++--- .../storage-postgres/src/stream_lifecycle.rs | 93 ++++++++++--- .../storage-postgres/tests/stream_disable.rs | 126 ++++++++++++++++++ crates/storage/src/stream_lineage.rs | 20 +++ crates/storage/src/stream_lineage_tests.rs | 27 ++++ 6 files changed, 341 insertions(+), 48 deletions(-) diff --git a/crates/storage-cassandra/src/stream_lifecycle.rs b/crates/storage-cassandra/src/stream_lifecycle.rs index d23b5f039..a86236dd5 100644 --- a/crates/storage-cassandra/src/stream_lifecycle.rs +++ b/crates/storage-cassandra/src/stream_lifecycle.rs @@ -25,14 +25,22 @@ //! still write a new-stream record below it, which no stream reads; that case //! stays with the deferred cross-instance consistency item in the design //! document. +//! +//! `UpdateTable` writes the catalog row first and changes the shards after it +//! ([`PendingStream`]), as on PostgreSQL and MongoDB. A disable whose close +//! failed leaves a disabled stream whose shards are still open: a retried +//! disable is refused and closes them, and an enable closes them too, since an +//! enable closes the open shards of every earlier stream of the table. An +//! enable whose shard insert failed leaves an enabled stream with no shards, +//! which no record belongs to, replaced by a disable and an enable. use cdrs_tokio::types::IntoRustByName; use extenddb_core::types::{StreamSpecification, StreamViewType}; use extenddb_storage::error::StorageError; use extenddb_storage::stream_lineage::{ SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, - enable_still_current, next_stream_label, open_shard_ids_of, parse_shard_id, shard_id, - stored_specification, stream_change, + enable_still_current, next_stream_label, open_shard_ids_before, open_shard_ids_of, + parse_shard_id, shard_id, stored_specification, stream_change, }; use crate::CassandraEngine; @@ -94,9 +102,10 @@ pub(crate) fn shard_range_of(physical: String, row: &cdrs_tokio::types::rows::Ro /// The `tables` columns an `UpdateTable` stream change sets, as text. pub(crate) type StreamColumns = Vec<(&'static str, String)>; -/// The shard changes of an applied `UpdateTable` stream change: close the -/// open shards of the stream the change read (`close`, the table's label at -/// that time), then, on an enable, create the new stream's shards. +/// The shard changes of an applied `UpdateTable` stream change: on a disable, +/// close the open shards of the stream the change read (`close`, the table's +/// label at that time); on an enable, close the open shards of every earlier +/// stream, then create the new stream's shards. #[must_use] pub(crate) struct PendingStream { /// The specification JSON and label the change was decided on; the @@ -129,14 +138,17 @@ impl PendingStream { table_name: &str, table_id: &str, ) -> Result<(), StorageError> { - if let Some(label) = &self.close { - engine - .close_stream_shards(account_ks, table_id, label) - .await?; - } let Some((label, view_type)) = self.open else { + if let Some(label) = &self.close { + engine + .close_stream_shards(account_ks, table_id, label) + .await?; + } return Ok(()); }; + engine + .close_streams_before(account_ks, table_id, &label) + .await?; // Each new shard starts one above the larger of a sequence number // drawn now from the clock this server's writers draw from and the // last ending number among the table's shards (the ones just closed @@ -337,9 +349,33 @@ impl CassandraEngine { account_ks: &str, table_id: &str, label: &str, + ) -> Result<(), StorageError> { + self.close_selected_shards(account_ks, table_id, label, open_shard_ids_of) + .await + } + + /// Close the open shards of every stream of the table older than `label` + /// (`stream_lineage::open_shard_ids_before`), as [`Self::close_stream_shards`] + /// closes one stream's. + async fn close_streams_before( + &self, + account_ks: &str, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + self.close_selected_shards(account_ks, table_id, label, open_shard_ids_before) + .await + } + + async fn close_selected_shards( + &self, + account_ks: &str, + table_id: &str, + label: &str, + select: fn(&str, &str, &[ShardRow]) -> Vec, ) -> Result<(), StorageError> { let rows = self.stream_shard_rows(account_ks, Some(table_id)).await?; - let ids = open_shard_ids_of(table_id, label, &rows); + let ids = select(table_id, label, &rows); let open = rows .into_iter() .filter(|r| ids.contains(&r.shard_id)) diff --git a/crates/storage-mongodb/src/stream_lifecycle.rs b/crates/storage-mongodb/src/stream_lifecycle.rs index da2d2a80c..272dc954e 100644 --- a/crates/storage-mongodb/src/stream_lifecycle.rs +++ b/crates/storage-mongodb/src/stream_lifecycle.rs @@ -10,7 +10,13 @@ //! in the data database. `UpdateTable` sets the specification and label on the //! table document first and changes the shards after that write //! ([`PendingStream`]), the order the PostgreSQL backend uses for the same -//! two-database split. +//! two-database split. A failure between the two leaves the table document's +//! answer in force. A disable whose close failed leaves a disabled stream whose +//! shards are still open: a retried disable is refused and closes them, and an +//! enable closes them too, since an enable closes the open shards of every +//! earlier stream of the table. An enable whose shard insert failed leaves an +//! enabled stream with no shards, which records nothing and is replaced by a +//! disable and an enable. use futures::TryStreamExt; use mongodb::bson::DateTime as BsonDateTime; @@ -21,8 +27,8 @@ use extenddb_core::types::{StreamSpecification, StreamViewType}; use extenddb_storage::error::StorageError; use extenddb_storage::stream_lineage::{ SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, - enable_still_current, next_stream_label, open_shard_ids_of, shard_id, stored_specification, - stream_change, + enable_still_current, next_stream_label, open_shard_ids_before, open_shard_ids_of, shard_id, + stored_specification, stream_change, }; use crate::MongoEngine; @@ -50,9 +56,10 @@ fn table_row(table_name: &str, doc: &Document) -> TableStreamRow { } } -/// The shard changes of an applied `UpdateTable` stream change: close the -/// open shards of the stream the change read (`close`, the table's label at -/// that time), then, on an enable, create the new stream's shards. +/// The shard changes of an applied `UpdateTable` stream change: on a disable, +/// close the open shards of the stream the change read (`close`, the table's +/// label at that time); on an enable, close the open shards of every earlier +/// stream, then create the new stream's shards. #[must_use] pub(crate) struct PendingStream { table_id: String, @@ -71,24 +78,28 @@ impl PendingStream { /// with this label, closes the shards it inserted. A disable written /// before the re-read is seen by it; one written after finds all four /// shards open and closes them itself; both closing is idempotent through - /// the `ending_sequence_number: null` filter. + /// the `ending_sequence_number: null` filter. A disable closes only the + /// stream it read, so one applying after an overlapping enable leaves the + /// new stream open. pub(crate) async fn apply( self, engine: &MongoEngine, account_id: &str, table_name: &str, ) -> Result<(), StorageError> { - if let Some(label) = &self.close { - engine.close_stream_shards(&self.table_id, label).await?; - } - if let Some((label, view_type)) = self.open { - engine - .init_stream_shards(&self.table_id, &label, view_type) - .await?; - let row = engine.stream_table_row(account_id, table_name).await?; - if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { - engine.close_stream_shards(&self.table_id, &label).await?; + let Some((label, view_type)) = self.open else { + if let Some(label) = &self.close { + engine.close_stream_shards(&self.table_id, label).await?; } + return Ok(()); + }; + engine.close_streams_before(&self.table_id, &label).await?; + engine + .init_stream_shards(&self.table_id, &label, view_type) + .await?; + let row = engine.stream_table_row(account_id, table_name).await?; + if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { + engine.close_stream_shards(&self.table_id, &label).await?; } Ok(()) } @@ -138,13 +149,31 @@ impl MongoEngine { &self, table_id: &str, label: &str, + ) -> Result<(), StorageError> { + self.close_selected_shards(table_id, label, open_shard_ids_of) + .await + } + + /// Close the open shards of every stream of the table older than `label` + /// (`stream_lineage::open_shard_ids_before`), as [`Self::close_stream_shards`] + /// closes one stream's. + async fn close_streams_before(&self, table_id: &str, label: &str) -> Result<(), StorageError> { + self.close_selected_shards(table_id, label, open_shard_ids_before) + .await + } + + async fn close_selected_shards( + &self, + table_id: &str, + label: &str, + select: fn(&str, &str, &[ShardRow]) -> Vec, ) -> Result<(), StorageError> { let shards_coll = self.data_db.collection::("stream_shards"); let records_coll = self.data_db.collection::("stream_records"); let rows = self .stream_shard_rows(std::slice::from_ref(&table_id.to_owned())) .await?; - let ids = open_shard_ids_of(table_id, label, &rows); + let ids = select(table_id, label, &rows); if ids.is_empty() { return Ok(()); } diff --git a/crates/storage-postgres/src/stream_lifecycle.rs b/crates/storage-postgres/src/stream_lifecycle.rs index b75967450..0a012ce75 100644 --- a/crates/storage-postgres/src/stream_lifecycle.rs +++ b/crates/storage-postgres/src/stream_lifecycle.rs @@ -9,16 +9,19 @@ //! data database, so a stream change is two commits. `UpdateTable` changes the //! catalog row (specification and label) in its transaction and changes the //! shards after that transaction commits ([`PendingStream`]). A failure between -//! the two leaves the catalog's answer in force: a disabled stream whose shards -//! are still open (closed by the next change of that stream), or an enabled stream with no -//! shards, which records nothing and is replaced by a disable and an enable. +//! the two leaves the catalog's answer in force. A disable whose close failed +//! leaves a disabled stream whose shards are still open: a retried disable is +//! refused and closes them, and an enable closes them too, since an enable +//! closes the open shards of every earlier stream of the table. An enable whose +//! shard insert failed leaves an enabled stream with no shards, which records +//! nothing and is replaced by a disable and an enable. use extenddb_core::types::{StreamSpecification, StreamViewType}; use extenddb_storage::error::StorageError; use extenddb_storage::stream_lineage::{ SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, - enable_still_current, next_stream_label, open_shard_ids_of, shard_id, stored_specification, - stream_change, + enable_still_current, next_stream_label, open_shard_ids_before, open_shard_ids_of, shard_id, + stored_specification, stream_change, }; use sqlx::PgPool; @@ -37,9 +40,10 @@ fn spec_of(value: Option) -> Result Self { + Self { + table_id: table_id.to_owned(), + close: Some(label.to_owned()), + open: None, + } + } + /// Run the shard changes once the catalog transaction has committed. /// /// The enable's shards are inserted after its catalog row is visible, so a @@ -58,7 +73,9 @@ impl PendingStream { /// label, closes the shards it inserted. A disable committed before the /// re-read is seen by it; one committed after finds all four rows open /// and closes them itself; both closing is idempotent through the - /// `ending_sequence_number IS NULL` guard. + /// `ending_sequence_number IS NULL` guard. A disable closes only the + /// stream it read, so one applying after an overlapping enable leaves the + /// new stream open. pub(crate) async fn apply( self, engine: &PostgresEngine, @@ -66,16 +83,17 @@ impl PendingStream { table_name: &str, ) -> Result<(), StorageError> { let data_pool = &engine.data_pool; - if let Some(label) = &self.close { - PostgresEngine::close_stream_shards(data_pool, &self.table_id, label).await?; - } - if let Some((label, view_type)) = self.open { - PostgresEngine::insert_stream_shards(data_pool, &self.table_id, &label, view_type) - .await?; - let row = engine.stream_table_row(account_id, table_name).await?; - if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { - PostgresEngine::close_stream_shards(data_pool, &self.table_id, &label).await?; + let Some((label, view_type)) = self.open else { + if let Some(label) = &self.close { + PostgresEngine::close_stream_shards(data_pool, &self.table_id, label).await?; } + return Ok(()); + }; + PostgresEngine::close_streams_before(data_pool, &self.table_id, &label).await?; + PostgresEngine::insert_stream_shards(data_pool, &self.table_id, &label, view_type).await?; + let row = engine.stream_table_row(account_id, table_name).await?; + if !enable_still_current(row.as_ref().map(|(t, _)| t), &label) { + PostgresEngine::close_stream_shards(data_pool, &self.table_id, &label).await?; } Ok(()) } @@ -176,6 +194,26 @@ impl PostgresEngine { data_pool: &PgPool, table_id: &str, label: &str, + ) -> Result<(), StorageError> { + Self::close_selected_shards(data_pool, table_id, label, open_shard_ids_of).await + } + + /// Close the open shards of every stream of the table older than `label` + /// (`stream_lineage::open_shard_ids_before`), as [`Self::close_stream_shards`] + /// closes one stream's. + async fn close_streams_before( + data_pool: &PgPool, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + Self::close_selected_shards(data_pool, table_id, label, open_shard_ids_before).await + } + + async fn close_selected_shards( + data_pool: &PgPool, + table_id: &str, + label: &str, + select: fn(&str, &str, &[ShardRow]) -> Vec, ) -> Result<(), StorageError> { let rows: Vec = sqlx::query_as( "SELECT table_id, shard_id, parent_shard_id, starting_sequence_number, \ @@ -186,7 +224,7 @@ impl PostgresEngine { .fetch_all(data_pool) .await .map_err(internal)?; - let ids = open_shard_ids_of(table_id, label, &shard_rows(rows)); + let ids = select(table_id, label, &shard_rows(rows)); if ids.is_empty() { return Ok(()); } @@ -384,6 +422,23 @@ impl PostgresEngine { .map_err(internal)?; Ok(shard_rows(rows)) } + + /// Apply the shard changes of a disable that read stream `label` of table + /// `table_id`, as a disable whose apply runs after an overlapping enable's + /// would. + #[cfg(feature = "test-hooks")] + #[doc(hidden)] + pub async fn apply_disable_shard_changes_for_tests( + &self, + account_id: &str, + table_name: &str, + table_id: &str, + label: &str, + ) -> Result<(), StorageError> { + PendingStream::disable_of(table_id, label) + .apply(self, account_id, table_name) + .await + } } fn shard_rows(rows: Vec) -> Vec { diff --git a/crates/storage-postgres/tests/stream_disable.rs b/crates/storage-postgres/tests/stream_disable.rs index 5ed84fbd0..f2563fff1 100644 --- a/crates/storage-postgres/tests/stream_disable.rs +++ b/crates/storage-postgres/tests/stream_disable.rs @@ -749,3 +749,129 @@ async fn a_write_racing_a_disable_lands_before_the_ending_number_or_not_at_all() ); s.cleanup().await; } + +async fn open_shard_ids(s: &Scratch, table_id: &str) -> Vec { + sqlx::query_scalar( + "SELECT shard_id FROM stream_shards \ + WHERE table_id = $1 AND ending_sequence_number IS NULL ORDER BY shard_id", + ) + .bind(table_id) + .fetch_all(&s.db) + .await + .expect("open shard ids") +} + +/// A disable whose close failed after its catalog write, then an enable whose +/// close of the label it read failed too: the table is enabled under a later +/// label with no shards of its own, and the earlier stream's shards are still +/// open. The next enable closes every earlier stream's open shards, whatever +/// label it read. +#[tokio::test] +async fn an_enable_closes_an_earlier_stream_left_open() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("leftopen", true)) + .await + .expect("create"); + activate(&s, "leftopen").await; + put(&s, "leftopen", "k0", StreamViewType::NewAndOldImages).await; + let table = describe_table(&s, "leftopen").await; + let first_arn = table.latest_stream_arn.clone().expect("stream arn"); + let first = describe_stream(&s, &first_arn).await; + let first_ids: Vec = first.shards.iter().map(|x| x.shard_id.clone()).collect(); + let later = + extenddb_storage::stream_lineage::next_stream_label(Some(first.stream_label.as_str())); + sqlx::query("UPDATE tables SET stream_label = $1 WHERE table_name = 'leftopen'") + .bind(&later) + .execute(&s.db) + .await + .expect("move the label on, as the enable's catalog write did"); + assert_eq!(open_shard_ids(&s, &table.table_id).await, first_ids); + + set_stream(&s, "leftopen", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let reenabled = set_stream( + &s, + "leftopen", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("enable"); + let new = describe_stream(&s, &reenabled.latest_stream_arn.expect("new arn")).await; + let new_ids: Vec = new.shards.iter().map(|x| x.shard_id.clone()).collect(); + assert_eq!( + open_shard_ids(&s, &table.table_id).await, + new_ids, + "only the new stream's shards are open" + ); + for id in &first_ids { + let (start, end): (String, Option) = sqlx::query_as( + "SELECT starting_sequence_number, ending_sequence_number \ + FROM stream_shards WHERE shard_id = $1", + ) + .bind(id) + .fetch_one(&s.db) + .await + .expect("shard row"); + let expected = last_record(&s, id).await.unwrap_or(start); + assert_eq!(end.as_deref(), Some(expected.as_str())); + } + s.cleanup().await; +} + +/// A disable that read stream A and applies its shard changes only after an +/// overlapping enable has closed A and created stream B: it closes A's +/// shards, which are closed already, and leaves B's open. +#[tokio::test] +async fn a_disable_applying_after_an_overlapping_enable_leaves_the_new_stream_open() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("overlap", true)) + .await + .expect("create"); + activate(&s, "overlap").await; + put(&s, "overlap", "k0", StreamViewType::NewAndOldImages).await; + let table = describe_table(&s, "overlap").await; + let old_label = table.latest_stream_label.clone().expect("label"); + set_stream(&s, "overlap", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let closed = describe_stream(&s, &table.latest_stream_arn.clone().expect("arn")).await; + let reenabled = set_stream( + &s, + "overlap", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("enable"); + let new = describe_stream(&s, &reenabled.latest_stream_arn.expect("new arn")).await; + let new_ids: Vec = new.shards.iter().map(|x| x.shard_id.clone()).collect(); + put(&s, "overlap", "k1", StreamViewType::KeysOnly).await; + + s.engine + .apply_disable_shard_changes_for_tests(ACCOUNT, "overlap", &table.table_id, &old_label) + .await + .expect("the late disable's shard changes"); + assert_eq!( + open_shard_ids(&s, &table.table_id).await, + new_ids, + "the new stream's four shards stay open" + ); + let old_again = describe_stream(&s, &closed.stream_arn).await; + for (a, b) in old_again.shards.iter().zip(&closed.shards) { + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number + ); + } + s.cleanup().await; +} diff --git a/crates/storage/src/stream_lineage.rs b/crates/storage/src/stream_lineage.rs index 877e0ab53..a545a9cad 100644 --- a/crates/storage/src/stream_lineage.rs +++ b/crates/storage/src/stream_lineage.rs @@ -437,6 +437,26 @@ pub fn open_shard_ids_of(table_id: &str, label: &str, shards: &[ShardRow]) -> Ve ids } +/// The ids of the open shards of every stream of table `table_id` older than +/// `label`: the labelled rows carrying an earlier label, and the legacy rows. +/// An enable creating stream `label` closes these, so a stream left open by a +/// close that failed after its catalog write is closed by the next enable, +/// whatever label that enable read. +#[must_use] +pub fn open_shard_ids_before(table_id: &str, label: &str, shards: &[ShardRow]) -> Vec { + let mut ids: Vec = shards + .iter() + .filter(|r| r.table_id == table_id && r.ending_sequence_number.is_none()) + .filter(|r| match parse_shard_id(table_id, &r.shard_id) { + Some((ShardStream::Labelled { label: l, .. }, _)) => l.as_str() < label, + Some((ShardStream::Legacy, _)) | None => true, + }) + .map(|r| r.shard_id.clone()) + .collect(); + ids.sort(); + ids +} + /// A stream's status: only the table's newest stream, while its specification /// is enabled, is `ENABLED` (`DISABLING` once the table is being deleted); /// every other stream of the table is `DISABLED`. diff --git a/crates/storage/src/stream_lineage_tests.rs b/crates/storage/src/stream_lineage_tests.rs index d99b7f718..94da3f2df 100644 --- a/crates/storage/src/stream_lineage_tests.rs +++ b/crates/storage/src/stream_lineage_tests.rs @@ -539,3 +539,30 @@ fn a_table_has_a_labelled_shard_once_an_enable_wrote_one() { }]; assert!(!has_labelled_shard(TID, &foreign)); } + +#[test] +fn an_enable_closes_the_open_shards_of_every_earlier_stream() { + const LATEST: &str = "2026-10-24T08:55:00.000"; + let mut rows: Vec = (0..SHARDS_PER_STREAM) + .map(|i| row(format!("shardId-{TID}-{i:016}"), None)) + .collect(); + rows.extend(stream_rows(OLD, StreamViewType::KeysOnly, None)); + rows.extend(stream_rows(NEW, StreamViewType::NewImage, Some("5"))); + rows.extend(stream_rows(LATEST, StreamViewType::NewImage, None)); + let mut expected: Vec = (0..SHARDS_PER_STREAM) + .map(|i| format!("shardId-{TID}-{i:016}")) + .collect(); + expected.extend( + stream_rows(OLD, StreamViewType::KeysOnly, None) + .into_iter() + .map(|r| r.shard_id), + ); + expected.sort(); + // The legacy rows and the open OLD rows; NEW's rows are closed and + // LATEST's are not earlier. + assert_eq!(open_shard_ids_before(TID, LATEST, &rows), expected); + assert!( + open_shard_ids_before(TID, OLD, &stream_rows(OLD, StreamViewType::KeysOnly, None)) + .is_empty() + ); +} From 25f959383c933a9edea8ee447afde99e37df0b0f Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:27:53 +0000 Subject: [PATCH 14/15] refactor(postgres): split the stream disable storage tests into two files tests/stream_disable.rs had grown to 877 lines. The scratch database and request helpers move to tests/common/stream_disable_fixture.rs, which both test files include; the lifecycle and race tests stay in stream_disable.rs, and the two tests of states an earlier release left (the name-based shards from before v0.1.14, and a disabled stream whose shards stayed open) move to stream_disable_legacy.rs. The test bodies are unchanged; the helpers gain `pub`. The PostgreSQL storage step in the integration workflow runs the new --test stream_disable_legacy. --- .github/workflows/integration.yml | 2 +- .../tests/common/stream_disable_fixture.rs | 254 +++++++++ .../storage-postgres/tests/stream_disable.rs | 491 +----------------- .../tests/stream_disable_legacy.rs | 261 ++++++++++ 4 files changed, 519 insertions(+), 489 deletions(-) create mode 100644 crates/storage-postgres/tests/common/stream_disable_fixture.rs create mode 100644 crates/storage-postgres/tests/stream_disable_legacy.rs diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c7cb2c458..6d043eb83 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -323,7 +323,7 @@ jobs: run: >- cargo test --release -p extenddb-storage-postgres --test vector_control_plane --test put_create_race --test twi_conflict - --test stream_shard_retention --test stream_shard_ownership --test stream_records_page_order --test stream_disable --test backup_restore + --test stream_shard_retention --test stream_shard_ownership --test stream_records_page_order --test stream_disable --test stream_disable_legacy --test backup_restore --test gsi_online_build --test gsi_stale_writer --test table_snapshot # The daemonized server logs to syslog; dump it so server-side failures diff --git a/crates/storage-postgres/tests/common/stream_disable_fixture.rs b/crates/storage-postgres/tests/common/stream_disable_fixture.rs new file mode 100644 index 000000000..d7f5f0a02 --- /dev/null +++ b/crates/storage-postgres/tests/common/stream_disable_fixture.rs @@ -0,0 +1,254 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +#![allow(dead_code)] + +//! The scratch database and request helpers the stream disable tests share. +//! +//! Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (host-only, e.g. +//! `postgres://user:pass@127.0.0.1:5432`); the tests skip otherwise. Each +//! test builds its own scratch database and drops it afterwards. + +pub use extenddb_core::expression::ExpressionMaps; +pub use extenddb_core::types::{ + AttributeDefinition, BillingMode, CreateTableInput, DescribeStreamInput, DescribeTableInput, + KeySchemaElement, KeyType, ScalarAttributeType, StreamDescription, StreamSpecification, + StreamStatus, StreamViewType, TableDescription, +}; +pub use extenddb_storage::error::StorageError; +pub use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, TableEngine}; +pub use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; +pub use sqlx::PgPool; +pub use sqlx::postgres::PgPoolOptions; + +pub const ACCOUNT: &str = "123456789012"; +pub const REGION: &str = "us-east-1"; + +pub struct Scratch { + pub engine: PostgresEngine, + pub db: PgPool, + pub admin: PgPool, + pub db_name: String, +} + +impl Scratch { + pub async fn cleanup(self) { + let Scratch { + engine, + db, + admin, + db_name, + } = self; + drop(engine); + db.close().await; + sqlx::query(&format!( + "DROP DATABASE IF EXISTS \"{db_name}\" WITH (FORCE)" + )) + .execute(&admin) + .await + .expect("drop the scratch database"); + admin.close().await; + } +} + +pub fn base_conn() -> Option { + let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; + (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) +} + +/// Every `.sql` file under `migrations/` and `data_migrations/`, each +/// directory in filename order. +pub fn shipped_migrations() -> Vec { + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut out = Vec::new(); + for dir in ["migrations", "data_migrations"] { + let mut files: Vec<_> = std::fs::read_dir(root.join(dir)) + .expect("read the migrations directory") + .map(|e| e.expect("directory entry").path()) + .filter(|p| p.extension().is_some_and(|e| e == "sql")) + .collect(); + files.sort(); + for f in files { + out.push(std::fs::read_to_string(&f).expect("read a migration file")); + } + } + out +} + +pub async fn scratch() -> Scratch { + let base = base_conn().expect("caller checks base_conn() first"); + let db_name = format!("eddb_sdis_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&format!("{base}/postgres")) + .await + .expect("connect to the postgres maintenance database"); + sqlx::query(&format!("CREATE DATABASE \"{db_name}\"")) + .execute(&admin) + .await + .expect("create the scratch database"); + let url = format!("{base}/{db_name}"); + let db = PgPoolOptions::new() + .max_connections(2) + .connect(&url) + .await + .expect("connect to the scratch database"); + for sql in shipped_migrations() { + sqlx::raw_sql(&sql) + .execute(&db) + .await + .expect("apply a shipped migration"); + } + // One scratch database stands in for both the catalog and the data + // database; drop the foreign key that cannot exist across the two. + sqlx::query("ALTER TABLE stream_shards DROP CONSTRAINT IF EXISTS stream_shards_table_id_fkey") + .execute(&db) + .await + .expect("match the two-database layout"); + sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") + .execute(&db) + .await + .expect("pin the control-plane delay to zero"); + sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") + .bind(ACCOUNT) + .bind(format!("acct-{db_name}")) + .execute(&db) + .await + .expect("seed the account row"); + let engine = PostgresEngine::new( + &PostgresConfig { + connection_string: url, + pool_size: 10, + max_item_size_bytes: 400_000, + }, + REGION, + ) + .await + .expect("open a PostgresEngine on the scratch database"); + Scratch { + engine, + db, + admin, + db_name, + } +} + +pub fn table_input(name: &str, stream: bool) -> CreateTableInput { + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "pk".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "pk".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + billing_mode: Some(BillingMode::PayPerRequest), + stream_specification: stream.then_some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::NewAndOldImages), + }), + ..Default::default() + } +} + +/// Finish the control-plane transition the worker would run. +pub async fn activate(s: &Scratch, name: &str) { + sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_name = $1") + .bind(name) + .execute(&s.db) + .await + .expect("activate"); +} + +pub async fn set_stream( + s: &Scratch, + name: &str, + spec: serde_json::Value, +) -> Result { + let input = serde_json::json!({"TableName": name, "StreamSpecification": spec}); + let out = s + .engine + .update_table(ACCOUNT, serde_json::from_value(input).expect("input")) + .await; + activate(s, name).await; + out +} + +pub async fn describe_table(s: &Scratch, name: &str) -> TableDescription { + s.engine + .describe_table( + ACCOUNT, + DescribeTableInput { + table_name: name.to_owned(), + }, + ) + .await + .expect("describe table") +} + +pub async fn describe_stream(s: &Scratch, arn: &str) -> StreamDescription { + s.engine + .describe_stream( + ACCOUNT, + &DescribeStreamInput { + stream_arn: arn.to_owned(), + limit: None, + exclusive_start_shard_id: None, + }, + ) + .await + .expect("describe stream") +} + +pub async fn put(s: &Scratch, name: &str, pk: &str, view_type: StreamViewType) { + let key_info = s + .engine + .table_key_info(ACCOUNT, name) + .await + .expect("key info"); + let item = serde_json::from_value(serde_json::json!({"pk": {"S": pk}})).expect("item"); + let capture = StreamCapture { + view_type, + user_identity: None, + region: REGION.into(), + }; + s.engine + .put_item( + &key_info, + item, + false, + None, + &ExpressionMaps::default(), + Some(&capture), + ) + .await + .expect("put"); +} + +pub async fn last_record(s: &Scratch, shard: &str) -> Option { + sqlx::query_scalar("SELECT MAX(sequence_number) FROM stream_records WHERE shard_id = $1") + .bind(shard) + .fetch_one(&s.db) + .await + .expect("last record") +} + +pub async fn record_count(s: &Scratch) -> i64 { + sqlx::query_scalar("SELECT COUNT(*) FROM stream_records") + .fetch_one(&s.db) + .await + .expect("count records") +} + +pub async fn open_shard_ids(s: &Scratch, table_id: &str) -> Vec { + sqlx::query_scalar( + "SELECT shard_id FROM stream_shards \ + WHERE table_id = $1 AND ending_sequence_number IS NULL ORDER BY shard_id", + ) + .bind(table_id) + .fetch_all(&s.db) + .await + .expect("open shard ids") +} diff --git a/crates/storage-postgres/tests/stream_disable.rs b/crates/storage-postgres/tests/stream_disable.rs index f2563fff1..2c17c9dcb 100644 --- a/crates/storage-postgres/tests/stream_disable.rs +++ b/crates/storage-postgres/tests/stream_disable.rs @@ -9,238 +9,10 @@ //! `postgres://user:pass@127.0.0.1:5432`); skips otherwise. Each test builds //! its own scratch database and drops it afterwards. -use extenddb_core::expression::ExpressionMaps; -use extenddb_core::types::{ - AttributeDefinition, BillingMode, CreateTableInput, DescribeStreamInput, DescribeTableInput, - KeySchemaElement, KeyType, ScalarAttributeType, StreamDescription, StreamSpecification, - StreamStatus, StreamViewType, TableDescription, -}; -use extenddb_storage::error::StorageError; -use extenddb_storage::{DataEngine, StreamCapture, StreamEngine, TableEngine}; -use extenddb_storage_postgres::{PostgresConfig, PostgresEngine}; -use sqlx::PgPool; -use sqlx::postgres::PgPoolOptions; +#[path = "common/stream_disable_fixture.rs"] +mod fixture; -const ACCOUNT: &str = "123456789012"; -const REGION: &str = "us-east-1"; - -struct Scratch { - engine: PostgresEngine, - db: PgPool, - admin: PgPool, - db_name: String, -} - -impl Scratch { - async fn cleanup(self) { - let Scratch { - engine, - db, - admin, - db_name, - } = self; - drop(engine); - db.close().await; - sqlx::query(&format!( - "DROP DATABASE IF EXISTS \"{db_name}\" WITH (FORCE)" - )) - .execute(&admin) - .await - .expect("drop the scratch database"); - admin.close().await; - } -} - -fn base_conn() -> Option { - let conn = std::env::var("EXTENDDB_TEST_PG_CONNECTION_STRING").ok()?; - (!conn.trim().is_empty()).then(|| conn.trim_end_matches('/').to_owned()) -} - -/// Every `.sql` file under `migrations/` and `data_migrations/`, each -/// directory in filename order. -fn shipped_migrations() -> Vec { - let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); - let mut out = Vec::new(); - for dir in ["migrations", "data_migrations"] { - let mut files: Vec<_> = std::fs::read_dir(root.join(dir)) - .expect("read the migrations directory") - .map(|e| e.expect("directory entry").path()) - .filter(|p| p.extension().is_some_and(|e| e == "sql")) - .collect(); - files.sort(); - for f in files { - out.push(std::fs::read_to_string(&f).expect("read a migration file")); - } - } - out -} - -async fn scratch() -> Scratch { - let base = base_conn().expect("caller checks base_conn() first"); - let db_name = format!("eddb_sdis_{}", uuid::Uuid::new_v4().simple())[..24].to_owned(); - let admin = PgPoolOptions::new() - .max_connections(1) - .connect(&format!("{base}/postgres")) - .await - .expect("connect to the postgres maintenance database"); - sqlx::query(&format!("CREATE DATABASE \"{db_name}\"")) - .execute(&admin) - .await - .expect("create the scratch database"); - let url = format!("{base}/{db_name}"); - let db = PgPoolOptions::new() - .max_connections(2) - .connect(&url) - .await - .expect("connect to the scratch database"); - for sql in shipped_migrations() { - sqlx::raw_sql(&sql) - .execute(&db) - .await - .expect("apply a shipped migration"); - } - // One scratch database stands in for both the catalog and the data - // database; drop the foreign key that cannot exist across the two. - sqlx::query("ALTER TABLE stream_shards DROP CONSTRAINT IF EXISTS stream_shards_table_id_fkey") - .execute(&db) - .await - .expect("match the two-database layout"); - sqlx::query("UPDATE settings SET value = '0' WHERE key = 'control_plane_delay_seconds'") - .execute(&db) - .await - .expect("pin the control-plane delay to zero"); - sqlx::query("INSERT INTO accounts (account_id, account_name) VALUES ($1, $2)") - .bind(ACCOUNT) - .bind(format!("acct-{db_name}")) - .execute(&db) - .await - .expect("seed the account row"); - let engine = PostgresEngine::new( - &PostgresConfig { - connection_string: url, - pool_size: 10, - max_item_size_bytes: 400_000, - }, - REGION, - ) - .await - .expect("open a PostgresEngine on the scratch database"); - Scratch { - engine, - db, - admin, - db_name, - } -} - -fn table_input(name: &str, stream: bool) -> CreateTableInput { - CreateTableInput { - table_name: name.to_owned(), - key_schema: vec![KeySchemaElement { - attribute_name: "pk".to_owned(), - key_type: KeyType::Hash, - }], - attribute_definitions: vec![AttributeDefinition { - attribute_name: "pk".to_owned(), - attribute_type: ScalarAttributeType::S, - }], - billing_mode: Some(BillingMode::PayPerRequest), - stream_specification: stream.then_some(StreamSpecification { - stream_enabled: true, - stream_view_type: Some(StreamViewType::NewAndOldImages), - }), - ..Default::default() - } -} - -/// Finish the control-plane transition the worker would run. -async fn activate(s: &Scratch, name: &str) { - sqlx::query("UPDATE tables SET table_status = 'ACTIVE' WHERE table_name = $1") - .bind(name) - .execute(&s.db) - .await - .expect("activate"); -} - -async fn set_stream( - s: &Scratch, - name: &str, - spec: serde_json::Value, -) -> Result { - let input = serde_json::json!({"TableName": name, "StreamSpecification": spec}); - let out = s - .engine - .update_table(ACCOUNT, serde_json::from_value(input).expect("input")) - .await; - activate(s, name).await; - out -} - -async fn describe_table(s: &Scratch, name: &str) -> TableDescription { - s.engine - .describe_table( - ACCOUNT, - DescribeTableInput { - table_name: name.to_owned(), - }, - ) - .await - .expect("describe table") -} - -async fn describe_stream(s: &Scratch, arn: &str) -> StreamDescription { - s.engine - .describe_stream( - ACCOUNT, - &DescribeStreamInput { - stream_arn: arn.to_owned(), - limit: None, - exclusive_start_shard_id: None, - }, - ) - .await - .expect("describe stream") -} - -async fn put(s: &Scratch, name: &str, pk: &str, view_type: StreamViewType) { - let key_info = s - .engine - .table_key_info(ACCOUNT, name) - .await - .expect("key info"); - let item = serde_json::from_value(serde_json::json!({"pk": {"S": pk}})).expect("item"); - let capture = StreamCapture { - view_type, - user_identity: None, - region: REGION.into(), - }; - s.engine - .put_item( - &key_info, - item, - false, - None, - &ExpressionMaps::default(), - Some(&capture), - ) - .await - .expect("put"); -} - -async fn last_record(s: &Scratch, shard: &str) -> Option { - sqlx::query_scalar("SELECT MAX(sequence_number) FROM stream_records WHERE shard_id = $1") - .bind(shard) - .fetch_one(&s.db) - .await - .expect("last record") -} - -async fn record_count(s: &Scratch) -> i64 { - sqlx::query_scalar("SELECT COUNT(*) FROM stream_records") - .fetch_one(&s.db) - .await - .expect("count records") -} +use fixture::*; #[tokio::test] async fn disable_closes_every_shard_and_re_enable_creates_a_new_stream() { @@ -407,252 +179,6 @@ async fn disable_on_a_table_that_never_had_a_stream_is_refused() { s.cleanup().await; } -/// A stream-enabled table created under v0.1.13 or earlier has four shard -/// rows named `shardId-{table_name}-{index:016}`. Its stream stays -/// describable, readable and listed after the upgrade, keeps its view type -/// once disabled, and is replaced by a labelled stream on re-enable. -#[tokio::test] -async fn a_stream_from_before_v0_1_14_survives_the_upgrade() { - if base_conn().is_none() { - eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); - return; - } - let s = scratch().await; - s.engine - .create_table(ACCOUNT, table_input("legacy", false)) - .await - .expect("create"); - activate(&s, "legacy").await; - let table_id = describe_table(&s, "legacy").await.table_id; - // What a pre-v0.1.14 release wrote: the label and specification on the - // catalog row, four name-based open shards in the data database. - let label = "2026-10-20T10:00:00.000"; - sqlx::query( - "UPDATE tables SET stream_label = $1, stream_specification = $2 \ - WHERE account_id = $3 AND table_name = 'legacy'", - ) - .bind(label) - .bind(serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"})) - .bind(ACCOUNT) - .execute(&s.db) - .await - .expect("seed the catalog row"); - for i in 0..4 { - sqlx::query( - "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ - VALUES ($1, $2, $3)", - ) - .bind(format!("shardId-legacy-{i:016}")) - .bind(&table_id) - .bind(format!("{:021}", 0)) - .execute(&s.db) - .await - .expect("seed a legacy shard"); - } - let arn = describe_table(&s, "legacy") - .await - .latest_stream_arn - .expect("the seeded stream arn"); - assert!(arn.ends_with(label)); - - put(&s, "legacy", "k", StreamViewType::NewImage).await; - let open = describe_stream(&s, &arn).await; - assert!(matches!(open.stream_status, StreamStatus::Enabled)); - assert_eq!(open.stream_view_type, StreamViewType::NewImage); - let ids: Vec<&str> = open.shards.iter().map(|x| x.shard_id.as_str()).collect(); - assert_eq!( - ids, - (0..4) - .map(|i| format!("shardId-legacy-{i:016}")) - .collect::>() - ); - s.engine - .validate_shard(ACCOUNT, &arn, "shardId-legacy-0000000000000001") - .await - .expect("a legacy shard belongs to the stream"); - assert_eq!(record_count(&s).await, 1); - let (listed, _) = s - .engine - .list_streams(ACCOUNT, Some("legacy"), 100, None) - .await - .expect("list"); - assert_eq!(listed.len(), 1); - assert_eq!(listed[0].stream_arn, arn); - - set_stream(&s, "legacy", serde_json::json!({"StreamEnabled": false})) - .await - .expect("disable"); - let closed = describe_stream(&s, &arn).await; - assert!(matches!(closed.stream_status, StreamStatus::Disabled)); - assert_eq!( - closed.stream_view_type, - StreamViewType::NewImage, - "a disabled legacy stream keeps its view type" - ); - assert!( - closed - .shards - .iter() - .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) - ); - // The backend keeps the disabled specification (with the view type) and - // the engine omits it from every response. - let stored = describe_table(&s, "legacy").await.stream_specification; - assert_eq!( - stored, - Some(StreamSpecification { - stream_enabled: false, - stream_view_type: Some(StreamViewType::NewImage), - }) - ); - let mut rendered = describe_table(&s, "legacy").await; - rendered.prepare_for_response(); - assert!(rendered.stream_specification.is_none()); - put(&s, "legacy", "dark", StreamViewType::NewImage).await; - assert_eq!( - record_count(&s).await, - 1, - "a write after the disable records nothing" - ); - - let re = set_stream( - &s, - "legacy", - serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), - ) - .await - .expect("re-enable"); - let new_arn = re.latest_stream_arn.expect("new arn"); - assert_ne!(new_arn, arn); - let new = describe_stream(&s, &new_arn).await; - assert!(matches!(new.stream_status, StreamStatus::Enabled)); - assert_eq!(new.shards.len(), 4); - assert!(new.shards.iter().all(|x| x.shard_id.len() == 61)); - // The enable closed the four legacy rows: none is still open. - let open_legacy: i64 = sqlx::query_scalar( - "SELECT COUNT(*) FROM stream_shards \ - WHERE shard_id LIKE 'shardId-legacy-%' AND ending_sequence_number IS NULL", - ) - .fetch_one(&s.db) - .await - .expect("count open legacy shards"); - assert_eq!( - open_legacy, 0, - "the enable closes the legacy stream's shards" - ); - s.cleanup().await; -} - -/// A stream disabled by a release that stored `{StreamEnabled: false}` and -/// left its four name-based shards open (the same state a close that failed -/// after its catalog write leaves): writes still record into it. A disable is -/// refused, as on the service, and closes the open shards first, so the next -/// write records nothing. -#[tokio::test] -async fn a_disable_refused_on_a_disabled_stream_closes_the_shards_it_left_open() { - if base_conn().is_none() { - eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); - return; - } - let s = scratch().await; - s.engine - .create_table(ACCOUNT, table_input("base", false)) - .await - .expect("create"); - activate(&s, "base").await; - let table_id = describe_table(&s, "base").await.table_id; - let label = "2026-10-20T10:00:00.000"; - sqlx::query( - "UPDATE tables SET stream_label = $1, stream_specification = $2 \ - WHERE account_id = $3 AND table_name = 'base'", - ) - .bind(label) - .bind(serde_json::json!({"StreamEnabled": false})) - .bind(ACCOUNT) - .execute(&s.db) - .await - .expect("seed the catalog row"); - for i in 0..4 { - sqlx::query( - "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ - VALUES ($1, $2, $3)", - ) - .bind(format!("shardId-base-{i:016}")) - .bind(&table_id) - .bind(format!("{:021}", 0)) - .execute(&s.db) - .await - .expect("seed an open shard"); - } - let open_shards = || async { - sqlx::query_scalar::<_, i64>( - "SELECT COUNT(*) FROM stream_shards WHERE ending_sequence_number IS NULL", - ) - .fetch_one(&s.db) - .await - .expect("count open shards") - }; - let arn = describe_table(&s, "base") - .await - .latest_stream_arn - .expect("the seeded stream arn"); - assert!(matches!( - describe_stream(&s, &arn).await.stream_status, - StreamStatus::Disabled - )); - put(&s, "base", "dark", StreamViewType::KeysOnly).await; - assert_eq!(record_count(&s).await, 1, "the open shards still record"); - - match set_stream(&s, "base", serde_json::json!({"StreamEnabled": false})).await { - Err(StorageError::Validation(m)) => { - assert_eq!(m, "Table has no stream to disable: TableName: base"); - } - other => panic!("{other:?}"), - } - assert_eq!( - open_shards().await, - 0, - "the refused disable closed the shards" - ); - let closed = describe_stream(&s, &arn).await; - assert!(matches!(closed.stream_status, StreamStatus::Disabled)); - for shard in &closed.shards { - let expected = last_record(&s, &shard.shard_id) - .await - .unwrap_or_else(|| shard.sequence_number_range.starting_sequence_number.clone()); - assert_eq!( - shard - .sequence_number_range - .ending_sequence_number - .as_deref(), - Some(expected.as_str()) - ); - } - put(&s, "base", "after", StreamViewType::KeysOnly).await; - assert_eq!( - record_count(&s).await, - 1, - "a write after the close records nothing" - ); - // The stored specification is untouched, and a second refusal has nothing - // left to close. - assert_eq!( - describe_table(&s, "base").await.stream_specification, - Some(StreamSpecification { - stream_enabled: false, - stream_view_type: None, - }) - ); - match set_stream(&s, "base", serde_json::json!({"StreamEnabled": false})).await { - Err(StorageError::Validation(m)) => { - assert_eq!(m, "Table has no stream to disable: TableName: base"); - } - other => panic!("{other:?}"), - } - assert_eq!(open_shards().await, 0); - s.cleanup().await; -} - /// Writers racing a disable: every record a write lands is in the closed /// stream at or below its shard's `EndingSequenceNumber`, or the write records /// nothing. The write path locks only its own shard row, `FOR KEY SHARE`, and @@ -750,17 +276,6 @@ async fn a_write_racing_a_disable_lands_before_the_ending_number_or_not_at_all() s.cleanup().await; } -async fn open_shard_ids(s: &Scratch, table_id: &str) -> Vec { - sqlx::query_scalar( - "SELECT shard_id FROM stream_shards \ - WHERE table_id = $1 AND ending_sequence_number IS NULL ORDER BY shard_id", - ) - .bind(table_id) - .fetch_all(&s.db) - .await - .expect("open shard ids") -} - /// A disable whose close failed after its catalog write, then an enable whose /// close of the label it read failed too: the table is enabled under a later /// label with no shards of its own, and the earlier stream's shards are still diff --git a/crates/storage-postgres/tests/stream_disable_legacy.rs b/crates/storage-postgres/tests/stream_disable_legacy.rs new file mode 100644 index 000000000..fc07646c2 --- /dev/null +++ b/crates/storage-postgres/tests/stream_disable_legacy.rs @@ -0,0 +1,261 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Streams an earlier release left behind: the name-based shards of a +//! stream created before v0.1.14, and a disabled stream whose shards stayed +//! open. +//! +//! Needs `EXTENDDB_TEST_PG_CONNECTION_STRING` (host-only, e.g. +//! `postgres://user:pass@127.0.0.1:5432`); skips otherwise. Each test builds +//! its own scratch database and drops it afterwards. + +#[path = "common/stream_disable_fixture.rs"] +mod fixture; + +use fixture::*; + +/// A stream-enabled table created under v0.1.13 or earlier has four shard +/// rows named `shardId-{table_name}-{index:016}`. Its stream stays +/// describable, readable and listed after the upgrade, keeps its view type +/// once disabled, and is replaced by a labelled stream on re-enable. +#[tokio::test] +async fn a_stream_from_before_v0_1_14_survives_the_upgrade() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("legacy", false)) + .await + .expect("create"); + activate(&s, "legacy").await; + let table_id = describe_table(&s, "legacy").await.table_id; + // What a pre-v0.1.14 release wrote: the label and specification on the + // catalog row, four name-based open shards in the data database. + let label = "2026-10-20T10:00:00.000"; + sqlx::query( + "UPDATE tables SET stream_label = $1, stream_specification = $2 \ + WHERE account_id = $3 AND table_name = 'legacy'", + ) + .bind(label) + .bind(serde_json::json!({"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"})) + .bind(ACCOUNT) + .execute(&s.db) + .await + .expect("seed the catalog row"); + for i in 0..4 { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES ($1, $2, $3)", + ) + .bind(format!("shardId-legacy-{i:016}")) + .bind(&table_id) + .bind(format!("{:021}", 0)) + .execute(&s.db) + .await + .expect("seed a legacy shard"); + } + let arn = describe_table(&s, "legacy") + .await + .latest_stream_arn + .expect("the seeded stream arn"); + assert!(arn.ends_with(label)); + + put(&s, "legacy", "k", StreamViewType::NewImage).await; + let open = describe_stream(&s, &arn).await; + assert!(matches!(open.stream_status, StreamStatus::Enabled)); + assert_eq!(open.stream_view_type, StreamViewType::NewImage); + let ids: Vec<&str> = open.shards.iter().map(|x| x.shard_id.as_str()).collect(); + assert_eq!( + ids, + (0..4) + .map(|i| format!("shardId-legacy-{i:016}")) + .collect::>() + ); + s.engine + .validate_shard(ACCOUNT, &arn, "shardId-legacy-0000000000000001") + .await + .expect("a legacy shard belongs to the stream"); + assert_eq!(record_count(&s).await, 1); + let (listed, _) = s + .engine + .list_streams(ACCOUNT, Some("legacy"), 100, None) + .await + .expect("list"); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].stream_arn, arn); + + set_stream(&s, "legacy", serde_json::json!({"StreamEnabled": false})) + .await + .expect("disable"); + let closed = describe_stream(&s, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + assert_eq!( + closed.stream_view_type, + StreamViewType::NewImage, + "a disabled legacy stream keeps its view type" + ); + assert!( + closed + .shards + .iter() + .all(|x| x.sequence_number_range.ending_sequence_number.is_some()) + ); + // The backend keeps the disabled specification (with the view type) and + // the engine omits it from every response. + let stored = describe_table(&s, "legacy").await.stream_specification; + assert_eq!( + stored, + Some(StreamSpecification { + stream_enabled: false, + stream_view_type: Some(StreamViewType::NewImage), + }) + ); + let mut rendered = describe_table(&s, "legacy").await; + rendered.prepare_for_response(); + assert!(rendered.stream_specification.is_none()); + put(&s, "legacy", "dark", StreamViewType::NewImage).await; + assert_eq!( + record_count(&s).await, + 1, + "a write after the disable records nothing" + ); + + let re = set_stream( + &s, + "legacy", + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .expect("re-enable"); + let new_arn = re.latest_stream_arn.expect("new arn"); + assert_ne!(new_arn, arn); + let new = describe_stream(&s, &new_arn).await; + assert!(matches!(new.stream_status, StreamStatus::Enabled)); + assert_eq!(new.shards.len(), 4); + assert!(new.shards.iter().all(|x| x.shard_id.len() == 61)); + // The enable closed the four legacy rows: none is still open. + let open_legacy: i64 = sqlx::query_scalar( + "SELECT COUNT(*) FROM stream_shards \ + WHERE shard_id LIKE 'shardId-legacy-%' AND ending_sequence_number IS NULL", + ) + .fetch_one(&s.db) + .await + .expect("count open legacy shards"); + assert_eq!( + open_legacy, 0, + "the enable closes the legacy stream's shards" + ); + s.cleanup().await; +} + +/// A stream disabled by a release that stored `{StreamEnabled: false}` and +/// left its four name-based shards open (the same state a close that failed +/// after its catalog write leaves): writes still record into it. A disable is +/// refused, as on the service, and closes the open shards first, so the next +/// write records nothing. +#[tokio::test] +async fn a_disable_refused_on_a_disabled_stream_closes_the_shards_it_left_open() { + if base_conn().is_none() { + eprintln!("SKIP: EXTENDDB_TEST_PG_CONNECTION_STRING is not set"); + return; + } + let s = scratch().await; + s.engine + .create_table(ACCOUNT, table_input("base", false)) + .await + .expect("create"); + activate(&s, "base").await; + let table_id = describe_table(&s, "base").await.table_id; + let label = "2026-10-20T10:00:00.000"; + sqlx::query( + "UPDATE tables SET stream_label = $1, stream_specification = $2 \ + WHERE account_id = $3 AND table_name = 'base'", + ) + .bind(label) + .bind(serde_json::json!({"StreamEnabled": false})) + .bind(ACCOUNT) + .execute(&s.db) + .await + .expect("seed the catalog row"); + for i in 0..4 { + sqlx::query( + "INSERT INTO stream_shards (shard_id, table_id, starting_sequence_number) \ + VALUES ($1, $2, $3)", + ) + .bind(format!("shardId-base-{i:016}")) + .bind(&table_id) + .bind(format!("{:021}", 0)) + .execute(&s.db) + .await + .expect("seed an open shard"); + } + let open_shards = || async { + sqlx::query_scalar::<_, i64>( + "SELECT COUNT(*) FROM stream_shards WHERE ending_sequence_number IS NULL", + ) + .fetch_one(&s.db) + .await + .expect("count open shards") + }; + let arn = describe_table(&s, "base") + .await + .latest_stream_arn + .expect("the seeded stream arn"); + assert!(matches!( + describe_stream(&s, &arn).await.stream_status, + StreamStatus::Disabled + )); + put(&s, "base", "dark", StreamViewType::KeysOnly).await; + assert_eq!(record_count(&s).await, 1, "the open shards still record"); + + match set_stream(&s, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!( + open_shards().await, + 0, + "the refused disable closed the shards" + ); + let closed = describe_stream(&s, &arn).await; + assert!(matches!(closed.stream_status, StreamStatus::Disabled)); + for shard in &closed.shards { + let expected = last_record(&s, &shard.shard_id) + .await + .unwrap_or_else(|| shard.sequence_number_range.starting_sequence_number.clone()); + assert_eq!( + shard + .sequence_number_range + .ending_sequence_number + .as_deref(), + Some(expected.as_str()) + ); + } + put(&s, "base", "after", StreamViewType::KeysOnly).await; + assert_eq!( + record_count(&s).await, + 1, + "a write after the close records nothing" + ); + // The stored specification is untouched, and a second refusal has nothing + // left to close. + assert_eq!( + describe_table(&s, "base").await.stream_specification, + Some(StreamSpecification { + stream_enabled: false, + stream_view_type: None, + }) + ); + match set_stream(&s, "base", serde_json::json!({"StreamEnabled": false})).await { + Err(StorageError::Validation(m)) => { + assert_eq!(m, "Table has no stream to disable: TableName: base"); + } + other => panic!("{other:?}"), + } + assert_eq!(open_shards().await, 0); + s.cleanup().await; +} From a42aeb77cffd1859c1259171323ee939a92b4e6a Mon Sep 17 00:00:00 2001 From: Lee Hannigan <29015211+LeeroyHannigan@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:41:32 +0000 Subject: [PATCH 15/15] refactor(streams): move the labelled shard checks and the Cassandra shard reads to their own files Three files this change adds had grown past 600 lines. Pure moves, by concern: - stream_lineage.rs (603 lines): labelled_shard_id_range, labelled_shard_id_pattern and has_labelled_shard move to stream_lineage_labelled.rs, re-exported from stream_lineage. - storage-cassandra stream_lifecycle.rs (635): the table and shard row reads (stream_table_row, stream_table_rows, stream_shard_rows and their row helper) move to stream_lifecycle_rows.rs, a child module. - storage-cassandra tests/direct/stream_disable.rs (634): the two tests of a closed stream's ending numbers against the next stream's start move to tests/direct/stream_endings.rs; the request helpers they share with stream_disable.rs become pub(crate). --- .../storage-cassandra/src/stream_lifecycle.rs | 129 +-------- .../src/stream_lifecycle_rows.rs | 138 ++++++++++ .../tests/direct/stream_disable.rs | 260 ++---------------- .../tests/direct/stream_endings.rs | 244 ++++++++++++++++ .../tests/direct_integration.rs | 2 + crates/storage/src/stream_lineage.rs | 40 +-- crates/storage/src/stream_lineage_labelled.rs | 43 +++ 7 files changed, 462 insertions(+), 394 deletions(-) create mode 100644 crates/storage-cassandra/src/stream_lifecycle_rows.rs create mode 100644 crates/storage-cassandra/tests/direct/stream_endings.rs create mode 100644 crates/storage/src/stream_lineage_labelled.rs diff --git a/crates/storage-cassandra/src/stream_lifecycle.rs b/crates/storage-cassandra/src/stream_lifecycle.rs index a86236dd5..7eef053ce 100644 --- a/crates/storage-cassandra/src/stream_lifecycle.rs +++ b/crates/storage-cassandra/src/stream_lifecycle.rs @@ -38,9 +38,9 @@ use cdrs_tokio::types::IntoRustByName; use extenddb_core::types::{StreamSpecification, StreamViewType}; use extenddb_storage::error::StorageError; use extenddb_storage::stream_lineage::{ - SHARDS_PER_STREAM, ShardRow, StreamChange, TableStreamRow, disabled_stream_to_close, - enable_still_current, next_stream_label, open_shard_ids_before, open_shard_ids_of, - parse_shard_id, shard_id, stored_specification, stream_change, + SHARDS_PER_STREAM, ShardRow, StreamChange, disabled_stream_to_close, enable_still_current, + next_stream_label, open_shard_ids_before, open_shard_ids_of, parse_shard_id, shard_id, + stored_specification, stream_change, }; use crate::CassandraEngine; @@ -492,128 +492,11 @@ impl CassandraEngine { }, }) } - - fn table_stream_row( - table_name: String, - row: &cdrs_tokio::types::rows::Row, - ) -> Result { - let spec: Option = row.get_by_name("stream_specification").ok().flatten(); - Ok(TableStreamRow { - table_name, - table_id: row.get_r_by_name("table_id").map_err(internal)?, - table_status: row.get_r_by_name("table_status").map_err(internal)?, - stream_label: row.get_by_name("stream_label").ok().flatten(), - stream_specification: spec - .as_deref() - .map(serde_json::from_str::) - .transpose() - .map_err(internal)?, - }) - } - - /// The table's stream columns and its key schema JSON, by name. - pub(crate) async fn stream_table_row( - &self, - account_id: &str, - table_name: &str, - ) -> Result, StorageError> { - let catalog_ks = self.catalog_keyspace(); - let row = query_rows::( - &self.session, - &format!( - "SELECT table_id, table_status, stream_label, stream_specification, key_schema \ - FROM {catalog_ks}.tables WHERE account_id = ? AND table_name = ?" - ), - cdrs_tokio::query_values!(account_id, table_name), - "stream_table_row", - ) - .await? - .into_iter() - .next(); - row.map(|r| { - Ok(( - Self::table_stream_row(table_name.to_owned(), &r)?, - r.get_r_by_name("key_schema").map_err(internal)?, - )) - }) - .transpose() - } - - /// Every table of the account that has had a stream. - pub(crate) async fn stream_table_rows( - &self, - account_id: &str, - ) -> Result, StorageError> { - let catalog_ks = self.catalog_keyspace(); - let rows = query_rows::( - &self.session, - &format!( - "SELECT table_name, table_id, table_status, stream_label, stream_specification \ - FROM {catalog_ks}.tables WHERE account_id = ? ALLOW FILTERING" - ), - cdrs_tokio::query_values!(account_id), - "stream_table_rows", - ) - .await?; - let mut tables = Vec::new(); - for r in rows { - let label: Option = r.get_by_name("stream_label").ok().flatten(); - if label.is_none() { - continue; - } - let name: String = r.get_r_by_name("table_name").map_err(internal)?; - tables.push(Self::table_stream_row(name, &r)?); - } - Ok(tables) - } - - /// The shard rows of one table, or of every table in the account's - /// keyspace when `table_id` is `None`. - pub(crate) async fn stream_shard_rows( - &self, - account_ks: &str, - table_id: Option<&str>, - ) -> Result, StorageError> { - let columns = "table_id, shard_id, parent_shard_id, starting_sequence_number, \ - ending_sequence_number"; - let rows = match table_id { - Some(id) => { - query_rows::( - &self.session, - &format!("SELECT {columns} FROM {account_ks}.stream_shards WHERE table_id = ?"), - cdrs_tokio::query_values!(id), - "stream_shard_rows", - ) - .await? - } - None => { - query_rows::( - &self.session, - &format!("SELECT {columns} FROM {account_ks}.stream_shards"), - cdrs_tokio::query::QueryValues::SimpleValues(Vec::new()), - "stream_shard_rows", - ) - .await? - } - }; - rows.into_iter() - .map(|r| { - Ok(ShardRow { - table_id: r.get_r_by_name("table_id").map_err(internal)?, - shard_id: r.get_r_by_name("shard_id").map_err(internal)?, - parent_shard_id: r.get_by_name("parent_shard_id").ok().flatten(), - starting_sequence_number: r - .get_by_name("starting_sequence_number") - .ok() - .flatten() - .unwrap_or_else(|| ZERO_SEQUENCE.to_owned()), - ending_sequence_number: r.get_by_name("ending_sequence_number").ok().flatten(), - }) - }) - .collect() - } } +#[path = "stream_lifecycle_rows.rs"] +mod rows; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/storage-cassandra/src/stream_lifecycle_rows.rs b/crates/storage-cassandra/src/stream_lifecycle_rows.rs new file mode 100644 index 000000000..fe23f12c1 --- /dev/null +++ b/crates/storage-cassandra/src/stream_lifecycle_rows.rs @@ -0,0 +1,138 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! The table rows and shard rows of the Cassandra stream lifecycle, read for +//! the shared rules in `extenddb_storage::stream_lineage` to group into +//! streams. + +use cdrs_tokio::types::IntoRustByName; +use extenddb_core::types::StreamSpecification; +use extenddb_storage::error::StorageError; +use extenddb_storage::stream_lineage::{ShardRow, TableStreamRow}; + +use super::internal; +use crate::CassandraEngine; +use crate::cassandra_util::query_rows; +use crate::stream_util::ZERO_SEQUENCE; + +impl CassandraEngine { + fn table_stream_row( + table_name: String, + row: &cdrs_tokio::types::rows::Row, + ) -> Result { + let spec: Option = row.get_by_name("stream_specification").ok().flatten(); + Ok(TableStreamRow { + table_name, + table_id: row.get_r_by_name("table_id").map_err(internal)?, + table_status: row.get_r_by_name("table_status").map_err(internal)?, + stream_label: row.get_by_name("stream_label").ok().flatten(), + stream_specification: spec + .as_deref() + .map(serde_json::from_str::) + .transpose() + .map_err(internal)?, + }) + } + + /// The table's stream columns and its key schema JSON, by name. + pub(crate) async fn stream_table_row( + &self, + account_id: &str, + table_name: &str, + ) -> Result, StorageError> { + let catalog_ks = self.catalog_keyspace(); + let row = query_rows::( + &self.session, + &format!( + "SELECT table_id, table_status, stream_label, stream_specification, key_schema \ + FROM {catalog_ks}.tables WHERE account_id = ? AND table_name = ?" + ), + cdrs_tokio::query_values!(account_id, table_name), + "stream_table_row", + ) + .await? + .into_iter() + .next(); + row.map(|r| { + Ok(( + Self::table_stream_row(table_name.to_owned(), &r)?, + r.get_r_by_name("key_schema").map_err(internal)?, + )) + }) + .transpose() + } + + /// Every table of the account that has had a stream. + pub(crate) async fn stream_table_rows( + &self, + account_id: &str, + ) -> Result, StorageError> { + let catalog_ks = self.catalog_keyspace(); + let rows = query_rows::( + &self.session, + &format!( + "SELECT table_name, table_id, table_status, stream_label, stream_specification \ + FROM {catalog_ks}.tables WHERE account_id = ? ALLOW FILTERING" + ), + cdrs_tokio::query_values!(account_id), + "stream_table_rows", + ) + .await?; + let mut tables = Vec::new(); + for r in rows { + let label: Option = r.get_by_name("stream_label").ok().flatten(); + if label.is_none() { + continue; + } + let name: String = r.get_r_by_name("table_name").map_err(internal)?; + tables.push(Self::table_stream_row(name, &r)?); + } + Ok(tables) + } + + /// The shard rows of one table, or of every table in the account's + /// keyspace when `table_id` is `None`. + pub(crate) async fn stream_shard_rows( + &self, + account_ks: &str, + table_id: Option<&str>, + ) -> Result, StorageError> { + let columns = "table_id, shard_id, parent_shard_id, starting_sequence_number, \ + ending_sequence_number"; + let rows = match table_id { + Some(id) => { + query_rows::( + &self.session, + &format!("SELECT {columns} FROM {account_ks}.stream_shards WHERE table_id = ?"), + cdrs_tokio::query_values!(id), + "stream_shard_rows", + ) + .await? + } + None => { + query_rows::( + &self.session, + &format!("SELECT {columns} FROM {account_ks}.stream_shards"), + cdrs_tokio::query::QueryValues::SimpleValues(Vec::new()), + "stream_shard_rows", + ) + .await? + } + }; + rows.into_iter() + .map(|r| { + Ok(ShardRow { + table_id: r.get_r_by_name("table_id").map_err(internal)?, + shard_id: r.get_r_by_name("shard_id").map_err(internal)?, + parent_shard_id: r.get_by_name("parent_shard_id").ok().flatten(), + starting_sequence_number: r + .get_by_name("starting_sequence_number") + .ok() + .flatten() + .unwrap_or_else(|| ZERO_SEQUENCE.to_owned()), + ending_sequence_number: r.get_by_name("ending_sequence_number").ok().flatten(), + }) + }) + .collect() + } +} diff --git a/crates/storage-cassandra/tests/direct/stream_disable.rs b/crates/storage-cassandra/tests/direct/stream_disable.rs index 4f59fa520..481e63223 100644 --- a/crates/storage-cassandra/tests/direct/stream_disable.rs +++ b/crates/storage-cassandra/tests/direct/stream_disable.rs @@ -21,7 +21,7 @@ use extenddb_storage_cassandra::CassandraEngine; use crate::helpers::setup_engine; -async fn activate(engine: &CassandraEngine) { +pub(crate) async fn activate(engine: &CassandraEngine) { tokio::time::sleep(Duration::from_millis(350)).await; engine .process_control_plane_transitions() @@ -29,7 +29,11 @@ async fn activate(engine: &CassandraEngine) { .expect("process table transitions"); } -async fn describe_table(engine: &CassandraEngine, account: &str, name: &str) -> TableDescription { +pub(crate) async fn describe_table( + engine: &CassandraEngine, + account: &str, + name: &str, +) -> TableDescription { engine .describe_table( account, @@ -41,7 +45,11 @@ async fn describe_table(engine: &CassandraEngine, account: &str, name: &str) -> .unwrap() } -async fn describe_stream(engine: &CassandraEngine, account: &str, arn: &str) -> StreamDescription { +pub(crate) async fn describe_stream( + engine: &CassandraEngine, + account: &str, + arn: &str, +) -> StreamDescription { engine .describe_stream( account, @@ -55,7 +63,7 @@ async fn describe_stream(engine: &CassandraEngine, account: &str, arn: &str) -> .unwrap() } -async fn set_stream( +pub(crate) async fn set_stream( engine: &CassandraEngine, account: &str, name: &str, @@ -69,7 +77,13 @@ async fn set_stream( out } -async fn put(engine: &CassandraEngine, account: &str, name: &str, pk: &str, view: StreamViewType) { +pub(crate) async fn put( + engine: &CassandraEngine, + account: &str, + name: &str, + pk: &str, + view: StreamViewType, +) { let key_info = engine.table_key_info(account, name).await.unwrap(); let mut item = BTreeMap::new(); item.insert("id".to_owned(), AttributeValue::S(pk.to_owned())); @@ -92,7 +106,12 @@ async fn put(engine: &CassandraEngine, account: &str, name: &str, pk: &str, view } /// Every record a shard of the stream `arn` yields, from its start. -async fn read_all(engine: &CassandraEngine, account: &str, arn: &str, shard: &str) -> Vec { +pub(crate) async fn read_all( + engine: &CassandraEngine, + account: &str, + arn: &str, + shard: &str, +) -> Vec { let (records, _) = engine .get_stream_records(account, arn, shard, None, 1000) .await @@ -403,232 +422,3 @@ async fn overlapping_stream_changes_apply_exactly_one() { activate(&engine).await; } } - -/// The closed stream's last record came from a server whose clock is ahead of -/// this one: the new stream starts above that ending number, not at this -/// server's clock, so the record stays in one stream. -#[tokio::test] -async fn a_new_stream_starts_above_the_closed_streams_ending_numbers() { - if crate::helpers::skip_without_cassandra() { - return; - } - let engine = setup_engine().await; - let account = crate::helpers::unique_test_account(); - crate::helpers::ensure_test_account(&engine, &account) - .await - .unwrap(); - let name = "StreamSkew"; - engine - .create_table( - &account, - CreateTableInput { - table_name: name.to_owned(), - key_schema: vec![KeySchemaElement { - attribute_name: "id".to_owned(), - key_type: KeyType::Hash, - }], - attribute_definitions: vec![AttributeDefinition { - attribute_name: "id".to_owned(), - attribute_type: ScalarAttributeType::S, - }], - stream_specification: Some(StreamSpecification { - stream_enabled: true, - stream_view_type: Some(StreamViewType::KeysOnly), - }), - ..Default::default() - }, - ) - .await - .unwrap(); - activate(&engine).await; - let old_arn = describe_table(&engine, &account, name) - .await - .latest_stream_arn - .unwrap(); - set_stream( - &engine, - &account, - name, - serde_json::json!({"StreamEnabled": false}), - ) - .await - .unwrap(); - let closed = describe_stream(&engine, &account, &old_arn).await; - // An ending number an hour ahead of this server's clock, as a record from - // a server that far ahead would leave it. - let ahead = format!( - "{:013}{:06}{:04}", - chrono::Utc::now().timestamp_millis() + 3_600_000, - 0, - 1 - ); - let account_ks = engine.account_keyspace(&account); - let update = format!( - "UPDATE {account_ks}.stream_shards SET ending_sequence_number = ? WHERE shard_id = ?" - ); - for shard in &closed.shards { - engine - .session_arc() - .query_with_values( - &update, - cdrs_tokio::query_values!(ahead.as_str(), shard.shard_id.as_str()), - ) - .await - .unwrap(); - } - - let reenabled = set_stream( - &engine, - &account, - name, - serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), - ) - .await - .unwrap(); - let new = describe_stream(&engine, &account, &reenabled.latest_stream_arn.unwrap()).await; - assert_eq!(new.shards.len(), 4); - for shard in &new.shards { - let start = &shard.sequence_number_range.starting_sequence_number; - assert_eq!(start.len(), ahead.len()); - assert!( - start.as_str() > ahead.as_str(), - "start {start} is not above the closed stream's ending number {ahead}" - ); - } -} - -/// A disable whose close read the old stream's shards as open and writes their -/// ending numbers only after an overlapping enable closed them, started the -/// new stream and the new stream took records: the old shards keep the ending -/// numbers the first close wrote, below the new stream's start. -#[tokio::test] -async fn a_late_close_keeps_the_ending_numbers_the_first_close_wrote() { - if crate::helpers::skip_without_cassandra() { - return; - } - let engine = setup_engine().await; - let account = crate::helpers::unique_test_account(); - crate::helpers::ensure_test_account(&engine, &account) - .await - .unwrap(); - let name = "StreamLateClose"; - engine - .create_table( - &account, - CreateTableInput { - table_name: name.to_owned(), - key_schema: vec![KeySchemaElement { - attribute_name: "id".to_owned(), - key_type: KeyType::Hash, - }], - attribute_definitions: vec![AttributeDefinition { - attribute_name: "id".to_owned(), - attribute_type: ScalarAttributeType::S, - }], - stream_specification: Some(StreamSpecification { - stream_enabled: true, - stream_view_type: Some(StreamViewType::KeysOnly), - }), - ..Default::default() - }, - ) - .await - .unwrap(); - activate(&engine).await; - let table = describe_table(&engine, &account, name).await; - let old_arn = table.latest_stream_arn.clone().unwrap(); - for i in 0..8 { - put( - &engine, - &account, - name, - &format!("a{i}"), - StreamViewType::KeysOnly, - ) - .await; - } - // The rows a disable reads before it closes: every shard open. - let read_open: Vec = - describe_stream(&engine, &account, &old_arn) - .await - .shards - .into_iter() - .map(|s| extenddb_storage::stream_lineage::ShardRow { - table_id: table.table_id.clone(), - shard_id: s.shard_id, - parent_shard_id: s.parent_shard_id, - starting_sequence_number: s.sequence_number_range.starting_sequence_number, - ending_sequence_number: None, - }) - .collect(); - assert_eq!(read_open.len(), 4); - - set_stream( - &engine, - &account, - name, - serde_json::json!({"StreamEnabled": false}), - ) - .await - .unwrap(); - let closed = describe_stream(&engine, &account, &old_arn).await; - let reenabled = set_stream( - &engine, - &account, - name, - serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), - ) - .await - .unwrap(); - let new_arn = reenabled.latest_stream_arn.unwrap(); - for i in 0..8 { - put( - &engine, - &account, - name, - &format!("b{i}"), - StreamViewType::KeysOnly, - ) - .await; - } - let new = describe_stream(&engine, &account, &new_arn).await; - let mut new_records = 0; - for shard in &new.shards { - new_records += read_all(&engine, &account, &new_arn, &shard.shard_id) - .await - .len(); - } - assert_eq!(new_records, 8, "the new stream holds the eight new records"); - - // The late close of the old stream's label. - engine - .close_shards_as_read( - &engine.account_keyspace(&account), - &table.table_id, - read_open, - ) - .await - .unwrap(); - let after = describe_stream(&engine, &account, &old_arn).await; - for (a, b) in after.shards.iter().zip(&closed.shards) { - assert_eq!(a.shard_id, b.shard_id); - assert_eq!( - a.sequence_number_range.ending_sequence_number, - b.sequence_number_range.ending_sequence_number, - "the late close moved shard {}'s ending number", - a.shard_id - ); - } - let start = &new.shards[0].sequence_number_range.starting_sequence_number; - for shard in &after.shards { - let end = shard - .sequence_number_range - .ending_sequence_number - .as_deref() - .unwrap(); - assert!( - end < start.as_str(), - "ending {end} is not below the new start {start}" - ); - } -} diff --git a/crates/storage-cassandra/tests/direct/stream_endings.rs b/crates/storage-cassandra/tests/direct/stream_endings.rs new file mode 100644 index 000000000..cd8a316ac --- /dev/null +++ b/crates/storage-cassandra/tests/direct/stream_endings.rs @@ -0,0 +1,244 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! A closed stream's ending numbers against the next stream's start on +//! Cassandra, where every stream of a table keeps its records in the same +//! partition per shard index. + +use extenddb_core::types::{ + AttributeDefinition, CreateTableInput, KeySchemaElement, KeyType, ScalarAttributeType, + StreamSpecification, StreamViewType, +}; +use extenddb_storage::TableEngine; + +use crate::helpers::setup_engine; +use crate::stream_disable::{activate, describe_stream, describe_table, put, read_all, set_stream}; + +/// The closed stream's last record came from a server whose clock is ahead of +/// this one: the new stream starts above that ending number, not at this +/// server's clock, so the record stays in one stream. +#[tokio::test] +async fn a_new_stream_starts_above_the_closed_streams_ending_numbers() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamSkew"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + let old_arn = describe_table(&engine, &account, name) + .await + .latest_stream_arn + .unwrap(); + set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let closed = describe_stream(&engine, &account, &old_arn).await; + // An ending number an hour ahead of this server's clock, as a record from + // a server that far ahead would leave it. + let ahead = format!( + "{:013}{:06}{:04}", + chrono::Utc::now().timestamp_millis() + 3_600_000, + 0, + 1 + ); + let account_ks = engine.account_keyspace(&account); + let update = format!( + "UPDATE {account_ks}.stream_shards SET ending_sequence_number = ? WHERE shard_id = ?" + ); + for shard in &closed.shards { + engine + .session_arc() + .query_with_values( + &update, + cdrs_tokio::query_values!(ahead.as_str(), shard.shard_id.as_str()), + ) + .await + .unwrap(); + } + + let reenabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new = describe_stream(&engine, &account, &reenabled.latest_stream_arn.unwrap()).await; + assert_eq!(new.shards.len(), 4); + for shard in &new.shards { + let start = &shard.sequence_number_range.starting_sequence_number; + assert_eq!(start.len(), ahead.len()); + assert!( + start.as_str() > ahead.as_str(), + "start {start} is not above the closed stream's ending number {ahead}" + ); + } +} + +/// A disable whose close read the old stream's shards as open and writes their +/// ending numbers only after an overlapping enable closed them, started the +/// new stream and the new stream took records: the old shards keep the ending +/// numbers the first close wrote, below the new stream's start. +#[tokio::test] +async fn a_late_close_keeps_the_ending_numbers_the_first_close_wrote() { + if crate::helpers::skip_without_cassandra() { + return; + } + let engine = setup_engine().await; + let account = crate::helpers::unique_test_account(); + crate::helpers::ensure_test_account(&engine, &account) + .await + .unwrap(); + let name = "StreamLateClose"; + engine + .create_table( + &account, + CreateTableInput { + table_name: name.to_owned(), + key_schema: vec![KeySchemaElement { + attribute_name: "id".to_owned(), + key_type: KeyType::Hash, + }], + attribute_definitions: vec![AttributeDefinition { + attribute_name: "id".to_owned(), + attribute_type: ScalarAttributeType::S, + }], + stream_specification: Some(StreamSpecification { + stream_enabled: true, + stream_view_type: Some(StreamViewType::KeysOnly), + }), + ..Default::default() + }, + ) + .await + .unwrap(); + activate(&engine).await; + let table = describe_table(&engine, &account, name).await; + let old_arn = table.latest_stream_arn.clone().unwrap(); + for i in 0..8 { + put( + &engine, + &account, + name, + &format!("a{i}"), + StreamViewType::KeysOnly, + ) + .await; + } + // The rows a disable reads before it closes: every shard open. + let read_open: Vec = + describe_stream(&engine, &account, &old_arn) + .await + .shards + .into_iter() + .map(|s| extenddb_storage::stream_lineage::ShardRow { + table_id: table.table_id.clone(), + shard_id: s.shard_id, + parent_shard_id: s.parent_shard_id, + starting_sequence_number: s.sequence_number_range.starting_sequence_number, + ending_sequence_number: None, + }) + .collect(); + assert_eq!(read_open.len(), 4); + + set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": false}), + ) + .await + .unwrap(); + let closed = describe_stream(&engine, &account, &old_arn).await; + let reenabled = set_stream( + &engine, + &account, + name, + serde_json::json!({"StreamEnabled": true, "StreamViewType": "KEYS_ONLY"}), + ) + .await + .unwrap(); + let new_arn = reenabled.latest_stream_arn.unwrap(); + for i in 0..8 { + put( + &engine, + &account, + name, + &format!("b{i}"), + StreamViewType::KeysOnly, + ) + .await; + } + let new = describe_stream(&engine, &account, &new_arn).await; + let mut new_records = 0; + for shard in &new.shards { + new_records += read_all(&engine, &account, &new_arn, &shard.shard_id) + .await + .len(); + } + assert_eq!(new_records, 8, "the new stream holds the eight new records"); + + // The late close of the old stream's label. + engine + .close_shards_as_read( + &engine.account_keyspace(&account), + &table.table_id, + read_open, + ) + .await + .unwrap(); + let after = describe_stream(&engine, &account, &old_arn).await; + for (a, b) in after.shards.iter().zip(&closed.shards) { + assert_eq!(a.shard_id, b.shard_id); + assert_eq!( + a.sequence_number_range.ending_sequence_number, + b.sequence_number_range.ending_sequence_number, + "the late close moved shard {}'s ending number", + a.shard_id + ); + } + let start = &new.shards[0].sequence_number_range.starting_sequence_number; + for shard in &after.shards { + let end = shard + .sequence_number_range + .ending_sequence_number + .as_deref() + .unwrap(); + assert!( + end < start.as_str(), + "ending {end} is not below the new start {start}" + ); + } +} diff --git a/crates/storage-cassandra/tests/direct_integration.rs b/crates/storage-cassandra/tests/direct_integration.rs index 50193c8fd..ef9bb572a 100644 --- a/crates/storage-cassandra/tests/direct_integration.rs +++ b/crates/storage-cassandra/tests/direct_integration.rs @@ -53,6 +53,8 @@ mod settings_store; mod snapshot_engine; #[path = "direct/stream_disable.rs"] mod stream_disable; +#[path = "direct/stream_endings.rs"] +mod stream_endings; #[path = "direct/stream_ownership.rs"] mod stream_ownership; #[path = "direct/streams.rs"] diff --git a/crates/storage/src/stream_lineage.rs b/crates/storage/src/stream_lineage.rs index a545a9cad..ad1f09fd6 100644 --- a/crates/storage/src/stream_lineage.rs +++ b/crates/storage/src/stream_lineage.rs @@ -291,42 +291,6 @@ pub fn parse_shard_id(table_id: &str, shard_id: &str) -> Option<(ShardStream, u3 Some((ShardStream::Legacy, suffix.parse().ok()?)) } -/// The byte-order range `[from, to)` that holds the labelled shard ids of -/// table `table_id` and none of its legacy ones, for the backends that write -/// 16-digit suffixes (SQLite and PostgreSQL). A labelled suffix starts with -/// the label's 13 milliseconds digits, not all zeros, so it sorts at or above -/// `0000000000001`; a legacy `{index:016}` suffix starts with 13 zeros and -/// sorts below. The upper end keeps the `shardId-{table_name}-{index:016}` -/// form out, which sorts anywhere relative to the table id. -#[must_use] -pub fn labelled_shard_id_range(table_id: &str) -> (String, String) { - ( - format!("shardId-{table_id}-0000000000001"), - format!("shardId-{table_id}."), - ) -} - -/// A regular expression matching the labelled shard ids of table `table_id` -/// and none of its legacy ones, for MongoDB, whose legacy ids have 12-digit -/// suffixes that no byte-order bound separates from a labelled one. Table ids -/// are UUIDs, so the id needs no escaping. -#[must_use] -pub fn labelled_shard_id_pattern(table_id: &str) -> String { - format!("^shardId-{table_id}-(?!0{{13}})[0-9]{{16}}$") -} - -/// Whether any of `shards` is a labelled shard of table `table_id`. -#[must_use] -pub fn has_labelled_shard(table_id: &str, shards: &[ShardRow]) -> bool { - shards.iter().any(|r| { - r.table_id == table_id - && matches!( - parse_shard_id(table_id, &r.shard_id), - Some((ShardStream::Labelled { .. }, _)) - ) - }) -} - /// One shard row as a backend stores it. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ShardRow { @@ -594,6 +558,10 @@ pub fn invalid_exclusive_start_stream_arn(arn: &str) -> StorageError { )) } +#[path = "stream_lineage_labelled.rs"] +mod labelled; +pub use labelled::{has_labelled_shard, labelled_shard_id_pattern, labelled_shard_id_range}; + #[path = "stream_lineage_list.rs"] mod list; pub use list::list_streams; diff --git a/crates/storage/src/stream_lineage_labelled.rs b/crates/storage/src/stream_lineage_labelled.rs new file mode 100644 index 000000000..d357e58e8 --- /dev/null +++ b/crates/storage/src/stream_lineage_labelled.rs @@ -0,0 +1,43 @@ +// Copyright 2026 ExtendDB contributors +// SPDX-License-Identifier: Apache-2.0 + +//! Which of a table's shard rows are labelled, for the backends that check it +//! before serving a legacy shard (`util::shard_in_stream`). + +use super::{ShardRow, ShardStream, parse_shard_id}; + +/// The byte-order range `[from, to)` that holds the labelled shard ids of +/// table `table_id` and none of its legacy ones, for the backends that write +/// 16-digit suffixes (SQLite and PostgreSQL). A labelled suffix starts with +/// the label's 13 milliseconds digits, not all zeros, so it sorts at or above +/// `0000000000001`; a legacy `{index:016}` suffix starts with 13 zeros and +/// sorts below. The upper end keeps the `shardId-{table_name}-{index:016}` +/// form out, which sorts anywhere relative to the table id. +#[must_use] +pub fn labelled_shard_id_range(table_id: &str) -> (String, String) { + ( + format!("shardId-{table_id}-0000000000001"), + format!("shardId-{table_id}."), + ) +} + +/// A regular expression matching the labelled shard ids of table `table_id` +/// and none of its legacy ones, for MongoDB, whose legacy ids have 12-digit +/// suffixes that no byte-order bound separates from a labelled one. Table ids +/// are UUIDs, so the id needs no escaping. +#[must_use] +pub fn labelled_shard_id_pattern(table_id: &str) -> String { + format!("^shardId-{table_id}-(?!0{{13}})[0-9]{{16}}$") +} + +/// Whether any of `shards` is a labelled shard of table `table_id`. +#[must_use] +pub fn has_labelled_shard(table_id: &str, shards: &[ShardRow]) -> bool { + shards.iter().any(|r| { + r.table_id == table_id + && matches!( + parse_shard_id(table_id, &r.shard_id), + Some((ShardStream::Labelled { .. }, _)) + ) + }) +}