From dc6ba1ac48200a738d74f2bab25b9d05dfe1d3bb Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Thu, 12 Nov 2026 21:47:18 -0700 Subject: [PATCH 1/5] Added the Stream crate with its errors, offsets, epochs and id checks. A Stream id and a Producer id are opaque bytes, since the caller builds both. The Store requires a non-empty Producer id of at most 4096 bytes, and a kind. --- Cargo.toml | 1 + crates/streams/Cargo.toml | 30 ++ crates/streams/src/digest.rs | 44 +++ crates/streams/src/epoch.rs | 145 ++++++++ crates/streams/src/error.rs | 172 ++++++++++ crates/streams/src/ids.rs | 442 +++++++++++++++++++++++++ crates/streams/src/lib.rs | 24 ++ crates/streams/src/offset.rs | 101 ++++++ crates/streams/testdata/digests.json | 92 +++++ crates/streams/testdata/epochs.json | 53 +++ crates/streams/testdata/fields.json | 25 ++ crates/streams/testdata/gen_vectors.py | 221 +++++++++++++ crates/streams/testdata/ids.json | 25 ++ crates/streams/testdata/offsets.json | 22 ++ crates/streams/tests/vectors.rs | 155 +++++++++ 15 files changed, 1552 insertions(+) create mode 100644 crates/streams/Cargo.toml create mode 100644 crates/streams/src/digest.rs create mode 100644 crates/streams/src/epoch.rs create mode 100644 crates/streams/src/error.rs create mode 100644 crates/streams/src/ids.rs create mode 100644 crates/streams/src/lib.rs create mode 100644 crates/streams/src/offset.rs create mode 100644 crates/streams/testdata/digests.json create mode 100644 crates/streams/testdata/epochs.json create mode 100644 crates/streams/testdata/fields.json create mode 100644 crates/streams/testdata/gen_vectors.py create mode 100644 crates/streams/testdata/ids.json create mode 100644 crates/streams/testdata/offsets.json create mode 100644 crates/streams/tests/vectors.rs diff --git a/Cargo.toml b/Cargo.toml index 419d80247..a68c7be8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/macros", "crates/sdk", "crates/workflow", + "crates/streams", "crates/sdk-core-c-bridge", "crates/changelog-release-notes", ] diff --git a/crates/streams/Cargo.toml b/crates/streams/Cargo.toml new file mode 100644 index 000000000..e40ec7f4e --- /dev/null +++ b/crates/streams/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "temporalio-streams" +version = "0.1.0" +edition = "2024" +rust-version = "1.88.0" +authors = ["Temporal Technologies Inc. "] +license-file = { workspace = true } +description = "The experimental Stream runtime that Temporal SDK bridges link" +homepage = "https://temporal.io/" +repository = "https://github.com/temporalio/sdk-rust" +keywords = ["temporal", "workflow", "streams"] +categories = ["development-tools"] + +[dependencies] +prost = { workspace = true } +prost-types = { workspace = true } +sha2 = "0.10" +thiserror = { workspace = true } +tonic = { workspace = true } + +[dependencies.temporalio-common] +path = "../common" +version = "~1.2.0" +default-features = false + +[dev-dependencies] +serde_json = { workspace = true } + +[lints] +workspace = true diff --git a/crates/streams/src/digest.rs b/crates/streams/src/digest.rs new file mode 100644 index 000000000..ef448b357 --- /dev/null +++ b/crates/streams/src/digest.rs @@ -0,0 +1,44 @@ +//! The content digest of a batch. Each dedupe entry stores it, so that a resend with the same +//! `sequence` and other content fails instead of being dropped without an error. The Store compares +//! the digest that this function gives, and `testdata/digests.json` has its expected values. + +use crate::proto::AppendRecord; +use sha2::{Digest, Sha256}; + +/// The length of a [batch_digest]. +pub const DIGEST_LEN: usize = 16; + +/// The first 16 bytes of a SHA-256 over the Records in order. Each part has its length before it, +/// so that two different batches never hash the same bytes. The metadata is hashed in key order, +/// since a map has no order on the wire. +pub fn batch_digest(records: &[AppendRecord]) -> [u8; DIGEST_LEN] { + let mut hasher = Sha256::new(); + for record in records { + // A Record with no body and one with an empty body are two different stored Records. + let Some(body) = &record.body else { + hasher.update([0]); + continue; + }; + hasher.update([1]); + let mut metadata: Vec<_> = body.metadata.iter().collect(); + metadata.sort(); + hasher.update((metadata.len() as u64).to_be_bytes()); + for (key, value) in metadata { + part(&mut hasher, key.as_bytes()); + part(&mut hasher, value); + } + part(&mut hasher, &body.data); + hasher.update((body.external_payloads.len() as u64).to_be_bytes()); + for external in &body.external_payloads { + hasher.update(external.size_bytes.to_be_bytes()); + } + } + let mut digest = [0; DIGEST_LEN]; + digest.copy_from_slice(&hasher.finalize()[..DIGEST_LEN]); + digest +} + +fn part(hasher: &mut Sha256, bytes: &[u8]) { + hasher.update((bytes.len() as u64).to_be_bytes()); + hasher.update(bytes); +} diff --git a/crates/streams/src/epoch.rs b/crates/streams/src/epoch.rs new file mode 100644 index 000000000..63378e2ac --- /dev/null +++ b/crates/streams/src/epoch.rs @@ -0,0 +1,145 @@ +//! Producer epochs and their order. +//! +//! A Store keeps an epoch as a key of fixed-width big-endian fields, in the order of comparison. +//! A byte comparison of two keys then orders the epochs field by field. `testdata/epochs.json` has +//! the shared vectors of the keys, in hex. + +use crate::{ + StreamError, StreamResult, + proto::{Producer, WorkflowTaskEpoch, producer::Kind}, +}; +use prost_types::Timestamp; +use std::cmp::Ordering; + +/// The kind of a Producer, which is the case of its epoch. The stored field names and keys hold +/// its number, so each kind keeps its number. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ProducerKind { + /// A Workflow run. + Workflow = 1, + /// A Local Activity. + LocalActivity = 2, + /// An Activity of a Workflow. + Activity = 3, + /// A client. + Client = 4, + /// A Standalone Activity. + StandaloneActivity = 5, +} + +impl From<&Kind> for ProducerKind { + fn from(kind: &Kind) -> Self { + match kind { + Kind::Workflow(_) => Self::Workflow, + Kind::LocalActivity(_) => Self::LocalActivity, + Kind::WorkflowActivity(_) => Self::Activity, + Kind::StandaloneActivity(_) => Self::StandaloneActivity, + Kind::Client(_) => Self::Client, + } + } +} + +/// Refuses a Producer with no kind case. +impl TryFrom<&Producer> for ProducerKind { + type Error = StreamError; + + fn try_from(producer: &Producer) -> StreamResult { + kind_of(producer).ok_or_else(|| StreamError::invalid("The Producer has no kind")) + } +} + +/// The kind case of a Producer, `None` when it has none. +pub fn kind_of(producer: &Producer) -> Option { + producer.kind.as_ref().map(ProducerKind::from) +} + +/// The length of a [task_key]. +pub const TASK_KEY_LEN: usize = 24; +/// The length of an [epoch_key]. +pub const EPOCH_KEY_LEN: usize = 40; + +// A negative field is clamped at zero, since its bytes would outrank every real epoch. +fn be64(value: i64) -> [u8; 8] { + value.max(0).to_be_bytes() +} + +fn be32(value: i32) -> [u8; 4] { + value.max(0).to_be_bytes() +} + +fn time_fields(time: Option<&Timestamp>) -> (i64, i32) { + time.map_or((0, 0), |time| (time.seconds, time.nanos)) +} + +/// The key of a Workflow Task attempt. It is the started event id, the attempt, and the started +/// seconds and nanos, in 8, 4, 8 and 4 bytes. The epoch keys of the run and of the task's Local +/// Activities start with it. +pub fn task_key(task: &WorkflowTaskEpoch) -> [u8; TASK_KEY_LEN] { + let (seconds, nanos) = time_fields(task.started_time.as_ref()); + let mut key = [0; TASK_KEY_LEN]; + key[..8].copy_from_slice(&be64(task.started_event_id)); + key[8..12].copy_from_slice(&be32(task.attempt)); + key[12..20].copy_from_slice(&be64(seconds)); + key[20..].copy_from_slice(&be32(nanos)); + key +} + +/// The key of an epoch. It is the [task_of] key, then the attempt's scheduled seconds and nanos +/// and the attempt, in 8, 4 and 4 bytes. A field that the case doesn't have is zero, and a Producer +/// with no case is all zeros. +pub fn epoch_key(kind: Option<&Kind>) -> [u8; EPOCH_KEY_LEN] { + let (scheduled, attempt) = match kind { + Some(Kind::LocalActivity(local)) => (None, local.epoch.map_or(0, |e| e.attempt)), + Some(Kind::WorkflowActivity(activity) | Kind::StandaloneActivity(activity)) => { + let epoch = activity.epoch; + ( + epoch.and_then(|e| e.attempt_scheduled_time), + epoch.map_or(0, |e| e.attempt), + ) + } + // A client has one epoch for its whole life, so nothing fences it. + Some(Kind::Client(_)) => (None, 0), + Some(Kind::Workflow(_)) | None => (None, 0), + }; + let (seconds, nanos) = time_fields(scheduled.as_ref()); + let mut key = [0; EPOCH_KEY_LEN]; + key[..TASK_KEY_LEN].copy_from_slice(&task_key(&task_of(kind))); + key[24..32].copy_from_slice(&be64(seconds)); + key[32..36].copy_from_slice(&be32(nanos)); + key[36..].copy_from_slice(&be32(attempt)); + key +} + +/// The Workflow Task of an epoch. It is a Workflow's own task, or the task that dispatched a Local +/// Activity, and zero for the other kinds. +pub fn task_of(kind: Option<&Kind>) -> WorkflowTaskEpoch { + match kind { + Some(Kind::Workflow(w)) => w.epoch.unwrap_or_default(), + Some(Kind::LocalActivity(local)) => local.epoch.and_then(|e| e.task).unwrap_or_default(), + _ => WorkflowTaskEpoch::default(), + } +} + +/// Orders two epochs field by field. +pub fn compare_epochs(a: Option<&Kind>, b: Option<&Kind>) -> Ordering { + epoch_key(a).cmp(&epoch_key(b)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto::{LocalActivityEpoch, LocalActivityProducer, producer::Kind}; + + // Without the clamp, the bytes of a negative field would outrank every real epoch. + #[test] + fn a_negative_field_compares_as_zero() { + let negative = Kind::LocalActivity(LocalActivityProducer { + epoch: Some(LocalActivityEpoch { + task: None, + attempt: -1, + }), + ..Default::default() + }); + assert_eq!(compare_epochs(Some(&negative), None), Ordering::Equal); + } +} diff --git a/crates/streams/src/error.rs b/crates/streams/src/error.rs new file mode 100644 index 000000000..527015956 --- /dev/null +++ b/crates/streams/src/error.rs @@ -0,0 +1,172 @@ +//! The error of every failed call, and its form for lang. + +use crate::proto::{StreamErrorDetails, StreamErrorReason}; +use prost::Message; +use tonic::Code; + +/// The result of every fallible call in the crate. +pub type StreamResult = Result; + +/// Why a call failed. Each reason has one status code and one case of `StreamErrorReason`, and lang +/// maps the reason to one error class. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Reason { + /// Nobody created the Stream. + NotFound, + /// The Stream's Retention or last-write TTL passed. + Expired, + /// The Stream is closed. + Closed, + /// The Producer's epoch is lower than the newest one. + Fenced, + /// The batch has a lower `sequence` than the last one of its epoch. + SequenceTooLow, + /// The batch has the `sequence` of the last one of its epoch, and other content. + SeqDivergent, + /// The Stream's owner doesn't allow this Producer to close it. + NotOwner, + /// A newer format version wrote the Stream. + FormatTooNew, + /// The offset is past the newest offset, for example after a failover lost the newest Records. + OffsetTooHigh, + /// A Record or a batch is over the Store's limit. + TooLarge, + /// The request is malformed. + Invalid, + /// The Store failed or did not answer. A retry can succeed. + StorageUnavailable, + /// The Store lacks a capability that the call needs. + Unsupported, + /// A create that doesn't use an existing Stream names a Stream that exists. + AlreadyExists, + /// The Store gave back data that isn't valid. A retry reads the same data. + StorageCorrupt, +} + +impl Reason { + /// The token that starts the error's text. Lang reads the reason from + /// `StreamErrorDetails.reason`, not from this text. + pub fn token(self) -> &'static str { + match self { + Reason::NotFound => "stream_not_found", + Reason::Expired => "stream_expired", + Reason::Closed => "stream_closed", + Reason::Fenced => "stream_fenced", + Reason::SequenceTooLow => "stream_sequence_too_low", + Reason::SeqDivergent => "seq_divergent", + Reason::NotOwner => "stream_not_owner", + Reason::FormatTooNew => "stream_format_too_new", + Reason::OffsetTooHigh => "stream_offset_too_high", + Reason::TooLarge => "stream_too_large", + Reason::Invalid => "stream_invalid", + Reason::StorageUnavailable => "stream_storage_unavailable", + Reason::Unsupported => "stream_unsupported", + Reason::AlreadyExists => "stream_already_exists", + Reason::StorageCorrupt => "stream_storage_corrupt", + } + } + + /// The reason in the error details. + pub fn proto(self) -> StreamErrorReason { + match self { + Reason::NotFound => StreamErrorReason::NotFound, + Reason::Expired => StreamErrorReason::Expired, + Reason::Closed => StreamErrorReason::Closed, + Reason::Fenced => StreamErrorReason::Fenced, + Reason::SequenceTooLow => StreamErrorReason::SequenceTooLow, + Reason::SeqDivergent => StreamErrorReason::SeqDivergent, + Reason::NotOwner => StreamErrorReason::NotOwner, + Reason::FormatTooNew => StreamErrorReason::FormatTooNew, + Reason::OffsetTooHigh => StreamErrorReason::OffsetTooHigh, + Reason::TooLarge => StreamErrorReason::TooLarge, + Reason::Invalid => StreamErrorReason::Invalid, + Reason::StorageUnavailable => StreamErrorReason::StorageUnavailable, + Reason::Unsupported => StreamErrorReason::Unsupported, + Reason::AlreadyExists => StreamErrorReason::AlreadyExists, + Reason::StorageCorrupt => StreamErrorReason::StorageCorrupt, + } + } + + /// The status code of the reason, as in the proposal's Appendix A. + pub fn code(self) -> Code { + match self { + Reason::NotFound | Reason::Expired => Code::NotFound, + Reason::Closed + | Reason::Fenced + | Reason::SequenceTooLow + | Reason::SeqDivergent + | Reason::NotOwner + | Reason::FormatTooNew => Code::FailedPrecondition, + Reason::TooLarge | Reason::Invalid => Code::InvalidArgument, + // A read from the offset works again once the Stream has Records up to it. + Reason::OffsetTooHigh => Code::OutOfRange, + Reason::StorageUnavailable => Code::Unavailable, + Reason::Unsupported => Code::Unimplemented, + Reason::AlreadyExists => Code::AlreadyExists, + Reason::StorageCorrupt => Code::DataLoss, + } + } +} + +/// A failed Stream call. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("{}: {message}", reason.token())] +pub struct StreamError { + /// Why the call failed. + pub reason: Reason, + /// What happened, for a log or the facade error. + pub message: String, +} + +impl StreamError { + /// Creates an error. + pub fn new(reason: Reason, message: impl Into) -> Self { + Self { + reason, + message: message.into(), + } + } + + /// A malformed request. + pub fn invalid(message: impl Into) -> Self { + Self::new(Reason::Invalid, message) + } + + /// A Store that failed or did not answer. + pub fn storage_unavailable(message: impl Into) -> Self { + Self::new(Reason::StorageUnavailable, message) + } + + /// Store data that isn't valid. + pub fn storage_corrupt(message: impl Into) -> Self { + Self::new(Reason::StorageCorrupt, message) + } + + /// The status code that lang raises the error with. + pub fn code(&self) -> Code { + self.reason.code() + } + + /// The encoded [StreamErrorDetails], which lang reads the reason from, as from the details of + /// a gRPC status. + pub fn details(&self) -> Vec { + StreamErrorDetails { + reason: self.reason.proto() as i32, + } + .encode_to_vec() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_error_carries_its_code_and_reason_to_lang() { + let error = StreamError::new(Reason::FormatTooNew, "format 2 is newer than 1"); + assert_eq!(error.code(), Code::FailedPrecondition); + let details = StreamErrorDetails::decode(error.details().as_slice()).unwrap(); + assert_eq!(details.reason(), StreamErrorReason::FormatTooNew); + } +} diff --git a/crates/streams/src/ids.rs b/crates/streams/src/ids.rs new file mode 100644 index 000000000..0f7dbf172 --- /dev/null +++ b/crates/streams/src/ids.rs @@ -0,0 +1,442 @@ +//! The checks of Stream ids and Producers, and the Redis key layout. A caller builds both kinds of +//! id, so the crate treats each as opaque bytes. `testdata/ids.json` has the shared vectors of the +//! key layout. +//! +//! ```text +//! :{}:meta the create key, the fence, dedupe and close state +//! :{}:log the log, one entry per Record +//! :{}:closed the entry that a close adds, which wakes a waiting read +//! ``` +//! +//! A key has no Temporal namespace, so a Stream id is unique in one Store and key prefix. The part +//! in braces is the Redis Cluster hash tag, so that the keys are in one slot for one script. Each +//! part is percent-encoded, so a `:` or a brace in an id can't make two Streams share a key. + +use crate::{ + StreamError, StreamResult, + epoch::ProducerKind, + proto::{Producer, WorkflowTaskEpoch, producer::Kind}, +}; + +/// The longest Stream id, in bytes. +pub const MAX_STREAM_ID_BYTES: usize = 4096; + +/// The longest Producer id, in bytes. A caller builds the id from names that Temporal limits to +/// 1000 characters, so an id with two of them still fits. A Store keeps the id in one meta field of +/// each Producer for as long as the Stream lives, and in the first entry of each of its batches. +pub const MAX_PRODUCER_ID_BYTES: usize = 4096; + +/// The key prefix when the Store options set none. +pub const DEFAULT_KEY_PREFIX: &str = "temporal-streams"; + +/// Percent-encodes every byte but the unreserved ones, as Python's `quote(text, safe="")` does. +pub fn encode_part(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~') { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +/// Refuses an empty or too long Stream id. +pub fn check_stream_id(id: &str) -> StreamResult<()> { + if id.is_empty() { + return Err(StreamError::invalid("The Stream id is empty")); + } + if id.len() > MAX_STREAM_ID_BYTES { + return Err(StreamError::invalid(format!( + "The Stream id has {} bytes, more than {MAX_STREAM_ID_BYTES}", + id.len() + ))); + } + Ok(()) +} + +/// Refuses a Producer with no id, an id over [MAX_PRODUCER_ID_BYTES], or no kind. The id is +/// opaque, so the Store requires no shape of it. +pub fn check_producer(producer: &Producer) -> StreamResult<()> { + if producer.id.is_empty() { + return Err(StreamError::invalid("The Producer id is empty")); + } + if producer.id.len() > MAX_PRODUCER_ID_BYTES { + return Err(StreamError::invalid(format!( + "The Producer id has {} bytes, more than {MAX_PRODUCER_ID_BYTES}", + producer.id.len() + ))); + } + ProducerKind::try_from(producer)?; + Ok(()) +} + +fn missing(what: &str) -> StreamResult<()> { + Err(StreamError::invalid(format!("The Producer has no {what}"))) +} + +fn at_least_one(attempt: i32, of: &str) -> StreamResult<()> { + if attempt < 1 { + return Err(StreamError::invalid(format!( + "The {of} attempt is {attempt}, and it must be 1 or more" + ))); + } + Ok(()) +} + +/// Refuses a Workflow Task epoch with no started event id, or with an attempt below 1. +fn check_task(task: &WorkflowTaskEpoch) -> StreamResult<()> { + // Without its started event, a later task would have the epoch of an earlier one. + if task.started_event_id <= 0 { + return Err(StreamError::invalid( + "The Workflow Task of the epoch has no started event id", + )); + } + at_least_one(task.attempt, "Workflow Task") +} + +/// Refuses a Producer that lacks a field the API requires. A Workflow Producer only closes a +/// Stream, so it may send no epoch, but an epoch that it sends must order. A Workflow Task's +/// started time stays optional, since Core leaves it empty when the server gave the task none. +pub fn check_required(producer: &Producer) -> StreamResult<()> { + match producer.kind.as_ref() { + Some(Kind::Workflow(workflow)) => { + if workflow.run_id.is_empty() { + return missing("run id"); + } + workflow.epoch.as_ref().map_or(Ok(()), check_task) + } + Some(Kind::LocalActivity(local)) => { + if local.run_id.is_empty() { + return missing("run id"); + } + let Some(epoch) = local.epoch.as_ref() else { + return missing("epoch"); + }; + let Some(task) = epoch.task.as_ref() else { + return missing("Workflow Task that dispatched it"); + }; + check_task(task)?; + at_least_one(epoch.attempt, "Local Activity") + } + Some(Kind::WorkflowActivity(activity) | Kind::StandaloneActivity(activity)) => { + let Some(epoch) = activity.epoch.as_ref() else { + return missing("epoch"); + }; + if epoch.attempt_scheduled_time.is_none() { + return missing("attempt scheduled time"); + } + at_least_one(epoch.attempt, "Activity") + } + Some(Kind::Client(_)) => Ok(()), + None => missing("kind"), + } +} + +/// The keys of one Stream. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StreamKeys { + base: String, +} + +impl StreamKeys { + /// The keys of `stream_id` under `prefix`. + pub fn new(prefix: &str, stream_id: &str) -> Self { + Self { + base: format!("{}:{{{}}}", encode_part(prefix), encode_part(stream_id)), + } + } + + /// The hash with the format version, the fence, dedupe and close state. + pub fn meta(&self) -> String { + format!("{}:meta", self.base) + } + + /// The log. + pub fn log(&self) -> String { + format!("{}:log", self.base) + } + + /// The key that a close adds one entry to. A close writes no log entry, so a waiting read also + /// waits on this key, and the close wakes it. + pub fn closed(&self) -> String { + format!("{}:closed", self.base) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + Reason, + proto::{ + ActivityEpoch, ActivityProducer, ClientProducer, LocalActivityEpoch, + LocalActivityProducer, WorkflowProducer, WorkflowTaskEpoch, + }, + }; + use prost_types::Timestamp; + + fn producer(kind: ProducerKind, id: &str, run_id: &str) -> Producer { + let run_id = run_id.to_string(); + let kind = match kind { + ProducerKind::Workflow => Kind::Workflow(WorkflowProducer { + run_id, + ..Default::default() + }), + ProducerKind::LocalActivity => Kind::LocalActivity(LocalActivityProducer { + run_id, + ..Default::default() + }), + ProducerKind::Activity => Kind::WorkflowActivity(ActivityProducer { + run_id, + ..Default::default() + }), + ProducerKind::StandaloneActivity => Kind::StandaloneActivity(ActivityProducer { + run_id, + ..Default::default() + }), + ProducerKind::Client => Kind::Client(ClientProducer::default()), + }; + Producer { + id: id.to_string(), + kind: Some(kind), + } + } + + #[test] + fn a_producer_id_is_any_non_empty_text_up_to_the_limit() { + for id in ["x", "wf:a|b", "client:{é} %", "a:b:c:d:e:f"] { + check_producer(&producer(ProducerKind::Client, id, "")).unwrap(); + } + check_producer(&producer( + ProducerKind::Workflow, + &"a".repeat(MAX_PRODUCER_ID_BYTES), + "r", + )) + .unwrap(); + let no_kind = Producer { + kind: None, + ..producer(ProducerKind::Workflow, "wf:x", "x") + }; + let too_long = producer( + ProducerKind::Client, + &"a".repeat(MAX_PRODUCER_ID_BYTES + 1), + "", + ); + let empty = producer(ProducerKind::Client, "", ""); + for bad in [no_kind, too_long, empty] { + assert_eq!( + check_producer(&bad).unwrap_err().reason, + Reason::Invalid, + "{bad:?}" + ); + } + } + + #[test] + fn a_stream_id_is_bounded() { + check_stream_id("a").unwrap(); + check_stream_id(&"a".repeat(MAX_STREAM_ID_BYTES)).unwrap(); + assert!(check_stream_id("").is_err()); + assert!(check_stream_id(&"a".repeat(MAX_STREAM_ID_BYTES + 1)).is_err()); + } + + /// A Local Activity Producer with every field the API requires. + fn local_activity() -> Producer { + Producer { + id: "la:r:k".to_string(), + kind: Some(Kind::LocalActivity(LocalActivityProducer { + run_id: "r".to_string(), + epoch: Some(LocalActivityEpoch { + task: Some(WorkflowTaskEpoch { + started_event_id: 3, + attempt: 1, + started_time: None, + }), + attempt: 1, + }), + })), + } + } + + /// An Activity Producer with every field the API requires. + fn activity() -> Producer { + Producer { + id: "act:r:a:s".to_string(), + kind: Some(Kind::WorkflowActivity(ActivityProducer { + run_id: "r".to_string(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: Some(Timestamp::default()), + attempt: 1, + }), + })), + } + } + + /// A client Producer with every field the API requires. + fn client() -> Producer { + Producer { + id: "client:n:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } + } + + fn local_epoch(producer: &mut Producer) -> &mut LocalActivityEpoch { + let Some(Kind::LocalActivity(local)) = producer.kind.as_mut() else { + unreachable!() + }; + local.epoch.as_mut().unwrap() + } + + fn activity_epoch(producer: &mut Producer) -> &mut ActivityEpoch { + let Some(Kind::WorkflowActivity(activity)) = producer.kind.as_mut() else { + unreachable!() + }; + activity.epoch.as_mut().unwrap() + } + + fn refused(producer: &Producer) -> String { + let error = check_required(producer).unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + error.message + } + + #[test] + fn a_producer_with_its_required_fields_passes() { + // A Workflow Producer only closes, with no epoch, and a standalone Activity has no run. + let workflow = producer(ProducerKind::Workflow, "wf:r", "r"); + let mut standalone = activity(); + let Some(Kind::WorkflowActivity(kind)) = standalone.kind.take() else { + unreachable!() + }; + standalone.kind = Some(Kind::StandaloneActivity(ActivityProducer { + run_id: String::new(), + ..kind + })); + for producer in [workflow, local_activity(), activity(), standalone, client()] { + check_required(&producer).unwrap(); + } + } + + #[test] + fn a_workflow_producer_needs_its_run_id() { + let workflow = producer(ProducerKind::Workflow, "wf:", ""); + assert!(refused(&workflow).contains("run id")); + } + + /// A Workflow Producer with `epoch`. + fn workflow_with(epoch: WorkflowTaskEpoch) -> Producer { + Producer { + id: "wf:r".to_string(), + kind: Some(Kind::Workflow(WorkflowProducer { + run_id: "r".to_string(), + epoch: Some(epoch), + })), + } + } + + #[test] + fn a_workflow_epoch_passes_with_its_required_fields() { + let epoch = WorkflowTaskEpoch { + started_event_id: 3, + attempt: 1, + started_time: None, + }; + check_required(&workflow_with(epoch)).unwrap(); + } + + #[test] + fn a_workflow_epoch_needs_its_started_event_id() { + let epoch = WorkflowTaskEpoch { + attempt: 1, + ..Default::default() + }; + assert!(refused(&workflow_with(epoch)).contains("started event id")); + } + + #[test] + fn a_workflow_epoch_attempt_is_1_or_more() { + let epoch = WorkflowTaskEpoch { + started_event_id: 3, + ..Default::default() + }; + assert!(refused(&workflow_with(epoch)).contains("Workflow Task attempt")); + } + + #[test] + fn a_local_activity_producer_needs_its_run_id() { + let mut local = local_activity(); + let Some(Kind::LocalActivity(kind)) = local.kind.as_mut() else { + unreachable!() + }; + kind.run_id.clear(); + assert!(refused(&local).contains("run id")); + } + + #[test] + fn a_local_activity_producer_needs_its_epoch() { + let mut local = local_activity(); + let Some(Kind::LocalActivity(kind)) = local.kind.as_mut() else { + unreachable!() + }; + kind.epoch = None; + assert!(refused(&local).contains("epoch")); + } + + #[test] + fn a_local_activity_epoch_needs_its_workflow_task() { + let mut local = local_activity(); + local_epoch(&mut local).task = None; + assert!(refused(&local).contains("Workflow Task")); + } + + #[test] + fn a_local_activity_task_needs_its_started_event_id() { + let mut local = local_activity(); + local_epoch(&mut local) + .task + .as_mut() + .unwrap() + .started_event_id = 0; + assert!(refused(&local).contains("Workflow Task")); + } + + #[test] + fn a_local_activity_task_attempt_is_1_or_more() { + let mut local = local_activity(); + local_epoch(&mut local).task.as_mut().unwrap().attempt = 0; + assert!(refused(&local).contains("Workflow Task attempt")); + } + + #[test] + fn a_local_activity_attempt_is_1_or_more() { + let mut local = local_activity(); + local_epoch(&mut local).attempt = 0; + assert!(refused(&local).contains("Local Activity attempt")); + } + + #[test] + fn an_activity_producer_needs_its_epoch() { + let mut act = activity(); + let Some(Kind::WorkflowActivity(kind)) = act.kind.as_mut() else { + unreachable!() + }; + kind.epoch = None; + assert!(refused(&act).contains("epoch")); + } + + #[test] + fn an_activity_epoch_needs_its_scheduled_time() { + let mut act = activity(); + activity_epoch(&mut act).attempt_scheduled_time = None; + assert!(refused(&act).contains("scheduled time")); + } + + #[test] + fn an_activity_attempt_is_1_or_more() { + let mut act = activity(); + activity_epoch(&mut act).attempt = 0; + assert!(refused(&act).contains("Activity attempt")); + } +} diff --git a/crates/streams/src/lib.rs b/crates/streams/src/lib.rs new file mode 100644 index 000000000..982d83cad --- /dev/null +++ b/crates/streams/src/lib.rs @@ -0,0 +1,24 @@ +#![warn(missing_docs)] + +//! The Stream runtime of Temporal SDKs. It is experimental. +//! +//! The crate serves the five operations of the Stream service on a Stream Store. A lang bridge +//! builds the runtime and calls it with an operation name and the encoded request. + +pub mod digest; +pub mod epoch; +mod error; +pub mod ids; +pub mod offset; + +pub use error::{Reason, StreamError, StreamResult}; + +/// The protos that lang and the crate exchange. `temporalio-protos` generates them, so a value +/// needs no conversion between the crates. +pub mod proto { + pub use temporalio_common::protos::temporal::api::{ + enums::v1::{StreamErrorReason, StreamIdConflictPolicy, StreamStatus}, + stream::v1::*, + streamservice::v1::*, + }; +} diff --git a/crates/streams/src/offset.rs b/crates/streams/src/offset.rs new file mode 100644 index 000000000..4b3f580b7 --- /dev/null +++ b/crates/streams/src/offset.rs @@ -0,0 +1,101 @@ +//! Offsets. An offset is the Redis entry id `-` packed as `ms << 20 | seq` in an `i64`. +//! It is ordered and unique in one Stream, and sparse. Zero is never an offset. +//! +//! On the wire the offset is opaque `bytes`. Lang only compares two offsets and resumes a read with +//! one, so the crate keeps the `i64` form inside and converts at the proto boundary with +//! [offset_to_wire] and [offset_from_wire]. + +use crate::{StreamError, StreamResult}; + +/// The number of bits of the entry id's sequence part. +pub const SEQ_BITS: u32 = 20; + +/// The wire form of an offset, for the read and append fields that lang reads as opaque `bytes`. +/// +/// Zero, which is "no offset", is the empty string, so it compares below every real offset. A real +/// offset is 8-byte big-endian, which keeps the order of the positive `i64` values. +pub fn offset_to_wire(offset: i64) -> Vec { + if offset <= 0 { + return Vec::new(); + } + (offset as u64).to_be_bytes().to_vec() +} + +/// The offset of a wire value that [offset_to_wire] wrote. The empty string is "no offset", zero. +/// +/// A value longer than 8 bytes is a bad request, since no offset has more. +pub fn offset_from_wire(bytes: &[u8]) -> StreamResult { + if bytes.is_empty() { + return Ok(0); + } + if bytes.len() > 8 { + return Err(StreamError::invalid(format!( + "A Stream offset has {} bytes, more than 8", + bytes.len() + ))); + } + let mut buf = [0u8; 8]; + buf[8 - bytes.len()..].copy_from_slice(bytes); + Ok(u64::from_be_bytes(buf) as i64) +} + +const SEQ_MASK: i64 = (1 << SEQ_BITS) - 1; +/// The largest millisecond part that fits, which is past the year 2200. +const MAX_MS: u64 = (i64::MAX >> SEQ_BITS) as u64; + +/// Packs a Redis entry id. +pub fn offset_of_entry_id(id: &str) -> StreamResult { + let bad = || StreamError::storage_corrupt(format!("The Store gave the entry id {id:?}")); + let (ms, seq) = id.split_once('-').ok_or_else(bad)?; + let ms: u64 = ms.parse().map_err(|_| bad())?; + let seq: u64 = seq.parse().map_err(|_| bad())?; + if ms > MAX_MS || seq > SEQ_MASK as u64 { + return Err(StreamError::storage_corrupt(format!( + "The entry id {id:?} doesn't fit in an offset" + ))); + } + Ok(((ms as i64) << SEQ_BITS) | seq as i64) +} + +/// Unpacks an offset into its Redis entry id. The offset must be above zero. +pub fn entry_id_of_offset(offset: i64) -> String { + debug_assert!(offset > 0); + format!("{}-{}", offset >> SEQ_BITS, offset & SEQ_MASK) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_wire_offset_keeps_the_order_and_empty_is_none() { + assert_eq!(offset_to_wire(0), Vec::::new()); + assert_eq!(offset_from_wire(&[]).unwrap(), 0); + for offset in [1, 42, (1 << SEQ_BITS) | 7, i64::MAX] { + assert_eq!(offset_from_wire(&offset_to_wire(offset)).unwrap(), offset); + } + // The byte order matches the i64 order, so lang compares two offsets as bytes. + assert!(offset_to_wire(1) > offset_to_wire(0)); + assert!(offset_to_wire(256) > offset_to_wire(255)); + // A right-aligned short value reads as the same number a full one would. + assert_eq!(offset_from_wire(&[1]).unwrap(), 1); + assert_eq!( + offset_from_wire(&[0, 0, 0, 0, 0, 0, 0, 0, 0]) + .unwrap_err() + .reason, + crate::Reason::Invalid + ); + } + + #[test] + fn an_entry_id_that_doesnt_fit_is_refused() { + for id in ["1-1048576", &format!("{}-0", MAX_MS + 1), "x"] { + let error = offset_of_entry_id(id).unwrap_err(); + assert_eq!(error.reason, crate::Reason::StorageCorrupt, "{id}"); + } + assert_eq!( + offset_of_entry_id(&format!("{MAX_MS}-{SEQ_MASK}")).unwrap(), + i64::MAX + ); + } +} diff --git a/crates/streams/testdata/digests.json b/crates/streams/testdata/digests.json new file mode 100644 index 000000000..d870b506a --- /dev/null +++ b/crates/streams/testdata/digests.json @@ -0,0 +1,92 @@ +[ + { + "name": "empty batch", + "records": [], + "digest": "e3b0c44298fc1c149afbf4c8996fb924" + }, + { + "name": "one record, no body", + "records": [ + {} + ], + "digest": "6e340b9cffb37a989ca544e6bb780a2c" + }, + { + "name": "one record, an empty body", + "records": [ + { + "data": "" + } + ], + "digest": "6bb0f5246ac6ee3fa6a6879c512f393f" + }, + { + "name": "one record", + "records": [ + { + "data": "68656c6c6f" + } + ], + "digest": "00cedb65fe85721791cf031124cc4ba2" + }, + { + "name": "metadata in key order", + "records": [ + { + "metadata": { + "encoding": "6a736f6e2f706c61696e", + "a": "00ff" + }, + "data": "7b2278223a317d" + } + ], + "digest": "eea3c073d22bc6bd16ffd5c5a18a2900" + }, + { + "name": "two records", + "records": [ + { + "data": "01" + }, + { + "data": "0203" + } + ], + "digest": "b75e10295b9753dbbc5774e454c9a7fb" + }, + { + "name": "external payloads", + "records": [ + { + "data": "01", + "external_payloads": [ + 10, + 20 + ] + } + ], + "digest": "381530dbc9c2fb0d5ddefbc99e72c6e1" + }, + { + "name": "metadata key and value split one way", + "records": [ + { + "metadata": { + "ab": "63" + } + } + ], + "digest": "be5bd276e8b11daeb753e3d3b6d6f93d" + }, + { + "name": "metadata key and value split another way", + "records": [ + { + "metadata": { + "a": "6263" + } + } + ], + "digest": "ddc7d1a81ecebb12548d3adae06622b8" + } +] diff --git a/crates/streams/testdata/epochs.json b/crates/streams/testdata/epochs.json new file mode 100644 index 000000000..a98d022a4 --- /dev/null +++ b/crates/streams/testdata/epochs.json @@ -0,0 +1,53 @@ +[ + { + "kind": "workflow", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 2, + "task_started_time": [ + 1760000000, + 5 + ] + }, + "key": "0000000000000008000000020000000068e778000000000500000000000000000000000000000000", + "task_key": "0000000000000008000000020000000068e7780000000005" + }, + { + "kind": "local_activity", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 1760000000, + 5 + ], + "attempt": 3 + }, + "key": "0000000000000008000000010000000068e778000000000500000000000000000000000000000003", + "task_key": "0000000000000008000000010000000068e7780000000005" + }, + { + "kind": "activity", + "epoch": { + "attempt_scheduled_time": [ + 1760000001, + 999999999 + ], + "attempt": 2 + }, + "key": "0000000000000000000000000000000000000000000000000000000068e778013b9ac9ff00000002", + "task_key": "000000000000000000000000000000000000000000000000" + }, + { + "kind": "client", + "epoch": {}, + "key": "00000000000000000000000000000000000000000000000000000000000000000000000000000000", + "task_key": "000000000000000000000000000000000000000000000000" + }, + { + "kind": "empty", + "epoch": {}, + "key": "00000000000000000000000000000000000000000000000000000000000000000000000000000000", + "task_key": "000000000000000000000000000000000000000000000000" + } +] diff --git a/crates/streams/testdata/fields.json b/crates/streams/testdata/fields.json new file mode 100644 index 000000000..d4427f501 --- /dev/null +++ b/crates/streams/testdata/fields.json @@ -0,0 +1,25 @@ +{ + "kind": 3, + "producer_id": "ingest|eu:1 é", + "producer": "p|3|ingest|eu:1 é", + "producer_values": [ + { + "name": "a batch with Records", + "fence": "0000000000000000000000000000000000000000000000000000000068e778013b9ac9ff00000002", + "sequence": 3, + "first": "1760000000123-4", + "last": "1760000000123-6", + "digest": "aca82bcf079b9b292bba1a09da05f527", + "value": "0000000000000000000000000000000000000000000000000000000068e778013b9ac9ff00000002000000000000000300000199c82cc07b0000000400000199c82cc07b00000006aca82bcf079b9b292bba1a09da05f527" + }, + { + "name": "an empty batch that wrote no entry", + "fence": "0000000000000000000000000000000000000000000000000000000068e778020000000000000003", + "sequence": 1, + "first": "", + "last": "", + "digest": "e3b0c44298fc1c149afbf4c8996fb924", + "value": "0000000000000000000000000000000000000000000000000000000068e7780200000000000000030000000000000001000000000000000000000000000000000000000000000000e3b0c44298fc1c149afbf4c8996fb924" + } + ] +} diff --git a/crates/streams/testdata/gen_vectors.py b/crates/streams/testdata/gen_vectors.py new file mode 100644 index 000000000..a1c030bb1 --- /dev/null +++ b/crates/streams/testdata/gen_vectors.py @@ -0,0 +1,221 @@ +"""Writes the shared test vectors of the Stream runtime. + +This is a second implementation of the layout, in Python, so the crate's tests check the Rust +code against it. The crate's README says which side reads each file. Run it from this directory: + + python3 gen_vectors.py +""" + +import hashlib +import json +from urllib.parse import quote + +SEQ_BITS = 20 + + +def part(text): + return quote(text, safe="") + + +def keys(prefix, stream_id): + base = f"{part(prefix)}:{{{part(stream_id)}}}" + return { + "meta": f"{base}:meta", + "log": f"{base}:log", + "closed": f"{base}:closed", + } + + +def be64(value): + return max(value, 0).to_bytes(8, "big") + + +def be32(value): + return max(value, 0).to_bytes(4, "big") + + +def task_key(started_event_id=0, attempt=0, started_time=(0, 0)): + """24 bytes, which compare as bytes in the order of the fields.""" + return be64(started_event_id) + be32(attempt) + be64(started_time[0]) + be32(started_time[1]) + + +def epoch_key(epoch): + """40 bytes, the task key and then the scheduled time and the attempt.""" + scheduled = epoch.get("attempt_scheduled_time", (0, 0)) + task = task_key( + epoch.get("task_started_event_id", 0), + epoch.get("task_attempt", 0), + tuple(epoch.get("task_started_time", (0, 0))), + ) + return task + be64(scheduled[0]) + be32(scheduled[1]) + be32(epoch.get("attempt", 0)) + + +def ids(): + """A Stream id is opaque, so the keys take its bytes, whatever they hold.""" + cases = [] + for prefix, stream_id in [ + ("temporal-streams", "agent-1:run-1:turn-3"), + ("p:x", "a%3Ab%2Fc:r%251:%C3%A9%20name"), + ("temporal-streams", "{x}:a|b é"), + ]: + cases.append({"prefix": prefix, "stream_id": stream_id, **keys(prefix, stream_id)}) + return {"streams": cases} + + +def offsets(): + cases = [] + for entry_id in ["0-1", "1-0", "1700000000000-0", "1700000000000-7", "1760000000123-1048575"]: + ms, seq = (int(x) for x in entry_id.split("-")) + cases.append({"entry_id": entry_id, "offset": (ms << SEQ_BITS) | seq}) + return cases + + +def epochs(): + cases = [ + {"kind": "workflow", "epoch": {"task_started_event_id": 8, "task_attempt": 2, + "task_started_time": [1760000000, 5]}}, + {"kind": "local_activity", "epoch": {"task_started_event_id": 8, "task_attempt": 1, + "task_started_time": [1760000000, 5], + "attempt": 3}}, + {"kind": "activity", "epoch": {"attempt_scheduled_time": [1760000001, 999999999], + "attempt": 2}}, + {"kind": "client", "epoch": {}}, + {"kind": "empty", "epoch": {}}, + ] + for case in cases: + case["key"] = epoch_key(case["epoch"]).hex() + epoch = case["epoch"] + case["task_key"] = task_key( + epoch.get("task_started_event_id", 0), + epoch.get("task_attempt", 0), + tuple(epoch.get("task_started_time", (0, 0))), + ).hex() + return cases + + +def entry_id(text): + """An entry id in a Producer field, the ms in 8 bytes and the seq in 4, zero for none.""" + if not text: + return bytes(12) + ms, seq = (int(x) for x in text.split("-")) + return ms.to_bytes(8, "big") + seq.to_bytes(4, "big") + + +def producer_value(fence, sequence, first, last, digest): + """The 88 bytes of a `p|` field.""" + return fence + sequence.to_bytes(8, "big") + entry_id(first) + entry_id(last) + digest + + +def producer_values(): + with_records = epoch_key({"attempt_scheduled_time": [1760000001, 999999999], "attempt": 2}) + empty = epoch_key({"attempt_scheduled_time": [1760000002, 0], "attempt": 3}) + cases = [] + for name, fence, sequence, first, last, records in [ + ("a batch with Records", with_records, 3, "1760000000123-4", "1760000000123-6", + [{"data": "01"}]), + ("an empty batch that wrote no entry", empty, 1, "", "", []), + ]: + batch_digest = bytes.fromhex(digest(records)) + value = producer_value(fence, sequence, first, last, batch_digest) + cases.append({ + "name": name, + "fence": fence.hex(), + "sequence": sequence, + "first": first, + "last": last, + "digest": batch_digest.hex(), + "value": value.hex(), + }) + return cases + + +def fields(): + """A Producer id is opaque, so its field name takes the bytes, also a separator in them.""" + producer = "ingest|eu:1 é" + return { + "kind": 3, + "producer_id": producer, + "producer": f"p|3|{producer}", + "producer_values": producer_values(), + } + + +def digest(records): + """The content digest of a batch, which a dedupe entry keeps, as the first 16 bytes of the + SHA-256.""" + out = hashlib.sha256() + + def part(data): + out.update(len(data).to_bytes(8, "big")) + out.update(data) + + for record in records: + # A record with none of these keys has no body. + if not {"metadata", "data", "external_payloads"} & record.keys(): + out.update(b"\x00") + continue + out.update(b"\x01") + metadata = record.get("metadata", {}) + metadata = sorted((key.encode(), bytes.fromhex(value)) for key, value in metadata.items()) + out.update(len(metadata).to_bytes(8, "big")) + for key, value in metadata: + part(key) + part(value) + part(bytes.fromhex(record.get("data", ""))) + external = record.get("external_payloads", []) + out.update(len(external).to_bytes(8, "big")) + for size in external: + out.update(size.to_bytes(8, "big", signed=True)) + return out.digest()[:16].hex() + + +def digests(): + """Batches as hex bytes, with the hex digest that the crate's `batch_digest` must compute.""" + cases = [ + {"name": "empty batch", "records": []}, + {"name": "one record, no body", "records": [{}]}, + {"name": "one record, an empty body", "records": [{"data": ""}]}, + {"name": "one record", "records": [{"data": b"hello".hex()}]}, + { + "name": "metadata in key order", + "records": [ + { + "metadata": {"encoding": b"json/plain".hex(), "a": "00ff"}, + "data": b'{"x":1}'.hex(), + } + ], + }, + {"name": "two records", "records": [{"data": "01"}, {"data": "0203"}]}, + {"name": "external payloads", "records": [{"data": "01", "external_payloads": [10, 20]}]}, + # The length prefixes give these two different digests, though their joined bytes are equal. + { + "name": "metadata key and value split one way", + "records": [{"metadata": {"ab": "63"}}], + }, + { + "name": "metadata key and value split another way", + "records": [{"metadata": {"a": "6263"}}], + }, + ] + for case in cases: + case["digest"] = digest(case["records"]) + assert cases[-1]["digest"] != cases[-2]["digest"] + assert cases[1]["digest"] != cases[2]["digest"] + return cases + + +def main(): + for name, value in [ + ("ids.json", ids()), + ("offsets.json", offsets()), + ("epochs.json", epochs()), + ("fields.json", fields()), + ("digests.json", digests()), + ]: + with open(name, "w") as out: + json.dump(value, out, indent=2, ensure_ascii=False) + out.write("\n") + + +if __name__ == "__main__": + main() diff --git a/crates/streams/testdata/ids.json b/crates/streams/testdata/ids.json new file mode 100644 index 000000000..7f85ad77b --- /dev/null +++ b/crates/streams/testdata/ids.json @@ -0,0 +1,25 @@ +{ + "streams": [ + { + "prefix": "temporal-streams", + "stream_id": "agent-1:run-1:turn-3", + "meta": "temporal-streams:{agent-1%3Arun-1%3Aturn-3}:meta", + "log": "temporal-streams:{agent-1%3Arun-1%3Aturn-3}:log", + "closed": "temporal-streams:{agent-1%3Arun-1%3Aturn-3}:closed" + }, + { + "prefix": "p:x", + "stream_id": "a%3Ab%2Fc:r%251:%C3%A9%20name", + "meta": "p%3Ax:{a%253Ab%252Fc%3Ar%25251%3A%25C3%25A9%2520name}:meta", + "log": "p%3Ax:{a%253Ab%252Fc%3Ar%25251%3A%25C3%25A9%2520name}:log", + "closed": "p%3Ax:{a%253Ab%252Fc%3Ar%25251%3A%25C3%25A9%2520name}:closed" + }, + { + "prefix": "temporal-streams", + "stream_id": "{x}:a|b é", + "meta": "temporal-streams:{%7Bx%7D%3Aa%7Cb%20%C3%A9}:meta", + "log": "temporal-streams:{%7Bx%7D%3Aa%7Cb%20%C3%A9}:log", + "closed": "temporal-streams:{%7Bx%7D%3Aa%7Cb%20%C3%A9}:closed" + } + ] +} diff --git a/crates/streams/testdata/offsets.json b/crates/streams/testdata/offsets.json new file mode 100644 index 000000000..c2f529e70 --- /dev/null +++ b/crates/streams/testdata/offsets.json @@ -0,0 +1,22 @@ +[ + { + "entry_id": "0-1", + "offset": 1 + }, + { + "entry_id": "1-0", + "offset": 1048576 + }, + { + "entry_id": "1700000000000-0", + "offset": 1782579200000000000 + }, + { + "entry_id": "1700000000000-7", + "offset": 1782579200000000007 + }, + { + "entry_id": "1760000000123-1048575", + "offset": 1845493760130023423 + } +] diff --git a/crates/streams/tests/vectors.rs b/crates/streams/tests/vectors.rs new file mode 100644 index 000000000..f6b423e59 --- /dev/null +++ b/crates/streams/tests/vectors.rs @@ -0,0 +1,155 @@ +//! The shared test vectors, which `testdata/gen_vectors.py` writes from a second implementation. +//! The crate's README says which other side reads each file. + +use prost_types::Timestamp; +use serde_json::Value; +use temporalio_common::protos::temporal::api::common::v1::{ + Payload, payload::ExternalPayloadDetails, +}; +use temporalio_streams::{ + digest::batch_digest, + epoch::{epoch_key, task_key, task_of}, + ids::StreamKeys, + offset::{entry_id_of_offset, offset_of_entry_id}, + proto::{ + ActivityEpoch, ActivityProducer, AppendRecord, ClientProducer, LocalActivityEpoch, + LocalActivityProducer, WorkflowProducer, WorkflowTaskEpoch, producer::Kind, + }, +}; + +fn load(name: &str) -> Value { + let path = format!("{}/testdata/{name}", env!("CARGO_MANIFEST_DIR")); + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +fn text<'a>(value: &'a Value, field: &str) -> &'a str { + value[field].as_str().unwrap() +} + +fn time(value: &Value) -> Option { + let pair = value.as_array()?; + Some(Timestamp { + seconds: pair[0].as_i64().unwrap(), + nanos: pair[1].as_i64().unwrap() as i32, + }) +} + +fn number(value: &Value, field: &str) -> i64 { + value[field].as_i64().unwrap_or(0) +} + +/// The Producer kind of a vector, from its flat fields and its run. The case takes the fields of +/// its kind, and a run-based kind carries the run. `empty` has no case. +fn kind_of(kind: &str, value: &Value, run_id: &str) -> Option { + let task = WorkflowTaskEpoch { + started_event_id: number(value, "task_started_event_id"), + attempt: number(value, "task_attempt") as i32, + started_time: time(&value["task_started_time"]), + }; + let attempt = number(value, "attempt") as i32; + let run_id = run_id.to_string(); + let activity = ActivityProducer { + run_id: run_id.clone(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: time(&value["attempt_scheduled_time"]), + attempt, + }), + }; + match kind { + "workflow" => Some(Kind::Workflow(WorkflowProducer { + run_id, + epoch: Some(task), + })), + "local_activity" => Some(Kind::LocalActivity(LocalActivityProducer { + run_id, + epoch: Some(LocalActivityEpoch { + task: Some(task), + attempt, + }), + })), + "activity" => Some(Kind::WorkflowActivity(activity)), + "standalone_activity" => Some(Kind::StandaloneActivity(activity)), + "client" => Some(Kind::Client(ClientProducer {})), + "empty" => None, + other => panic!("unknown kind {other}"), + } +} + +#[test] +fn stream_ids_and_keys_match() { + let vectors = load("ids.json"); + for case in vectors["streams"].as_array().unwrap() { + let keys = StreamKeys::new(text(case, "prefix"), text(case, "stream_id")); + assert_eq!(keys.meta(), text(case, "meta")); + assert_eq!(keys.log(), text(case, "log")); + assert_eq!(keys.closed(), text(case, "closed")); + } +} + +#[test] +fn offsets_match() { + for case in load("offsets.json").as_array().unwrap() { + let offset = number(case, "offset"); + assert_eq!(offset_of_entry_id(text(case, "entry_id")).unwrap(), offset); + assert_eq!(entry_id_of_offset(offset), text(case, "entry_id")); + } +} + +#[test] +fn epoch_keys_match() { + for case in load("epochs.json").as_array().unwrap() { + let kind = kind_of(text(case, "kind"), &case["epoch"], ""); + assert_eq!(epoch_key(kind.as_ref()).to_vec(), hex(text(case, "key"))); + assert_eq!( + task_key(&task_of(kind.as_ref())).to_vec(), + hex(text(case, "task_key")) + ); + } +} + +fn hex(text: &str) -> Vec { + (0..text.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&text[i..i + 2], 16).unwrap()) + .collect() +} + +#[test] +fn batch_digests_match() { + for case in load("digests.json").as_array().unwrap() { + let records: Vec<_> = case["records"] + .as_array() + .unwrap() + .iter() + .map(|r| AppendRecord { + // A record with none of these keys has no body. + body: ["metadata", "data", "external_payloads"] + .iter() + .any(|key| r.get(key).is_some()) + .then(|| Payload { + metadata: r["metadata"] + .as_object() + .into_iter() + .flatten() + .map(|(key, value)| (key.clone(), hex(value.as_str().unwrap()))) + .collect(), + data: hex(r["data"].as_str().unwrap_or_default()), + external_payloads: r["external_payloads"] + .as_array() + .into_iter() + .flatten() + .map(|size| ExternalPayloadDetails { + size_bytes: size.as_i64().unwrap(), + }) + .collect(), + }), + }) + .collect(); + assert_eq!( + batch_digest(&records).to_vec(), + hex(text(case, "digest")), + "{}", + text(case, "name") + ); + } +} From cf78dd75d5a71ae631ebc4a769aaec34799c31c5 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Thu, 12 Nov 2026 21:56:37 -0700 Subject: [PATCH 2/5] Added the Store trait, the memory Store and the shared Store suite. A Store crate runs the suite on its own Store through `testing::TestStore`. The suite checks that a Producer id is stored as bytes, also with a field separator in it. --- crates/streams/Cargo.toml | 13 + crates/streams/src/lib.rs | 4 + crates/streams/src/memory.rs | 711 ++++++++++++++++++++++++ crates/streams/src/store.rs | 285 ++++++++++ crates/streams/src/store/lazy.rs | 181 +++++++ crates/streams/src/testing/fields.rs | 32 ++ crates/streams/src/testing/mod.rs | 142 +++++ crates/streams/src/testing/store.rs | 743 ++++++++++++++++++++++++++ crates/streams/src/testing/support.rs | 274 ++++++++++ 9 files changed, 2385 insertions(+) create mode 100644 crates/streams/src/memory.rs create mode 100644 crates/streams/src/store.rs create mode 100644 crates/streams/src/store/lazy.rs create mode 100644 crates/streams/src/testing/fields.rs create mode 100644 crates/streams/src/testing/mod.rs create mode 100644 crates/streams/src/testing/store.rs create mode 100644 crates/streams/src/testing/support.rs diff --git a/crates/streams/Cargo.toml b/crates/streams/Cargo.toml index e40ec7f4e..677c94f7e 100644 --- a/crates/streams/Cargo.toml +++ b/crates/streams/Cargo.toml @@ -11,11 +11,20 @@ repository = "https://github.com/temporalio/sdk-rust" keywords = ["temporal", "workflow", "streams"] categories = ["development-tools"] +[features] +# The shared suites, for the crates of other Stores. +test-support = ["dep:futures-util", "dep:serde_json"] + [dependencies] +async-trait = "0.1" +futures-util = { version = "0.3", default-features = false, features = ["std"], optional = true } +parking_lot = "0.12" prost = { workspace = true } prost-types = { workspace = true } +serde_json = { workspace = true, optional = true } sha2 = "0.10" thiserror = { workspace = true } +tokio = { version = "1.47", default-features = false, features = ["rt", "sync", "time"] } tonic = { workspace = true } [dependencies.temporalio-common] @@ -24,7 +33,11 @@ version = "~1.2.0" default-features = false [dev-dependencies] +futures-util = { version = "0.3", default-features = false, features = ["std"] } serde_json = { workspace = true } +tokio = { version = "1.47", features = ["macros", "rt-multi-thread", "time"] } +# The integration tests call the memory Store's test hooks, which test-support gates. +temporalio-streams = { path = ".", features = ["test-support"] } [lints] workspace = true diff --git a/crates/streams/src/lib.rs b/crates/streams/src/lib.rs index 982d83cad..747256328 100644 --- a/crates/streams/src/lib.rs +++ b/crates/streams/src/lib.rs @@ -9,7 +9,11 @@ pub mod digest; pub mod epoch; mod error; pub mod ids; +pub mod memory; pub mod offset; +pub mod store; +#[cfg(any(test, feature = "test-support"))] +pub mod testing; pub use error::{Reason, StreamError, StreamResult}; diff --git a/crates/streams/src/memory.rs b/crates/streams/src/memory.rs new file mode 100644 index 000000000..d9f1356dd --- /dev/null +++ b/crates/streams/src/memory.rs @@ -0,0 +1,711 @@ +//! The memory Store, for tests. It keeps the same fields as the Redis layout and runs the steps of +//! the Lua scripts in the same order, so that both Stores return the same results. Each call holds +//! one lock, since Redis runs one script at a time. + +use crate::{ + Reason, StreamError, StreamResult, + digest::DIGEST_LEN, + epoch::{EPOCH_KEY_LEN, ProducerKind, epoch_key}, + ids::StreamKeys, + offset::{entry_id_of_offset, offset_of_entry_id}, + proto::{Producer, StreamRef}, + store::{ + Appended, Batch, Capabilities, Closed, ConflictPolicy, Create, CreateOutcome, Created, + Entry, FORMAT_VERSION, MAX_SEQUENCE, Store, StreamState, TOMBSTONE, + }, +}; +use parking_lot::Mutex; +use std::{ + collections::{BTreeMap, HashMap}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::{sync::Notify, time::Instant}; + +/// The number of sequence numbers in one millisecond, since an offset packs them in 20 bits. +const SEQ_LIMIT: u64 = 1 << crate::offset::SEQ_BITS; + +#[derive(Clone, Default)] +struct Fields { + record: Option>, +} + +/// One Stream's meta hash and log, each with an expiry time, as Redis keys have. +#[derive(Default)] +struct Keys { + /// The meta hash, whose field names and values are bytes as in Redis. + meta: HashMap, Vec>, + meta_expires: i64, + /// `None` when the log key doesn't exist. + log: Option>, + log_expires: i64, + /// The newest id of the log, which Redis drops with the expired key. + last_id: (u64, u64), +} + +/// A Store in process memory, with the rules of the Redis scripts. +pub struct MemoryStore { + prefix: String, + retention: Duration, + last_write_ttl: Option, + streams: Mutex>, + appended: Notify, +} + +impl std::fmt::Debug for MemoryStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryStore") + .field("retention", &self.retention) + .field("last_write_ttl", &self.last_write_ttl) + .finish_non_exhaustive() + } +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_millis() as i64) +} + +fn millis(duration: Duration) -> i64 { + i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) +} + +fn refuse(reason: Reason, message: &str) -> StreamError { + StreamError::new(reason, message) +} + +fn id_text((ms, seq): (u64, u64)) -> String { + format!("{ms}-{seq}") +} + +fn parse_id(text: &str) -> Option<(u64, u64)> { + let (ms, seq) = text.split_once('-')?; + Some((ms.parse().ok()?, seq.parse().ok()?)) +} + +/// The length of a [ProducerField] value. +const PRODUCER_FIELD_LEN: usize = EPOCH_KEY_LEN + 8 + 2 * 12 + DIGEST_LEN; + +/// An entry id as a [ProducerField] keeps it, the ms and the seq. +pub(crate) type EntryId = (u64, u32); + +/// The `p||` field, one per Producer. Its value is 88 bytes. They hold the fence +/// and the last batch's sequence (8), first and last entry ids (12 each, the ms in 8 and the seq in +/// 4, zeros for none) and digest. Numbers are big-endian. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ProducerField { + /// The newest epoch key of the Producer, which is also the epoch of its last batch. + pub(crate) fence: [u8; EPOCH_KEY_LEN], + /// The last batch's sequence. + pub(crate) sequence: i64, + /// The last batch's first and last entry ids, `None` when it wrote no entry. + pub(crate) entries: Option<(EntryId, EntryId)>, + /// The last batch's digest. + pub(crate) digest: [u8; DIGEST_LEN], +} + +impl ProducerField { + pub(crate) fn encode(&self) -> Vec { + let mut value = Vec::with_capacity(PRODUCER_FIELD_LEN); + value.extend_from_slice(&self.fence); + value.extend_from_slice(&self.sequence.to_be_bytes()); + let (first, last) = self.entries.unwrap_or_default(); + for (ms, seq) in [first, last] { + value.extend_from_slice(&ms.to_be_bytes()); + value.extend_from_slice(&seq.to_be_bytes()); + } + value.extend_from_slice(&self.digest); + value + } +} + +/// Refuses a value of another length, which a later format may write. +impl TryFrom<&[u8]> for ProducerField { + type Error = StreamError; + + fn try_from(value: &[u8]) -> StreamResult { + let unknown = || { + refuse( + Reason::StorageCorrupt, + "the Producer field has an unknown format", + ) + }; + if value.len() != PRODUCER_FIELD_LEN { + return Err(unknown()); + } + let (fence, rest) = value.split_at(EPOCH_KEY_LEN); + let (sequence, rest) = rest.split_at(8); + let (first, rest) = rest.split_at(12); + let (last, digest) = rest.split_at(12); + let entry_id = |bytes: &[u8]| -> StreamResult { + let (ms, seq) = bytes.split_at(8); + let ms = ms.try_into().map_err(|_| unknown())?; + let seq = seq.try_into().map_err(|_| unknown())?; + Ok((u64::from_be_bytes(ms), u32::from_be_bytes(seq))) + }; + // A batch has both entry ids or neither. + let entries = match (entry_id(first)?, entry_id(last)?) { + ((0, 0), (0, 0)) => None, + ((0, 0), _) | (_, (0, 0)) => return Err(unknown()), + ids => Some(ids), + }; + Ok(Self { + fence: fence.try_into().map_err(|_| unknown())?, + sequence: i64::from_be_bytes(sequence.try_into().map_err(|_| unknown())?), + entries, + digest: digest.try_into().map_err(|_| unknown())?, + }) + } +} + +/// The [EntryId] of an entry id's text, `-`. +fn entry_id(text: &str) -> StreamResult { + let bad = || refuse(Reason::StorageCorrupt, "an entry id isn't -"); + let (ms, seq) = text.split_once('-').ok_or_else(bad)?; + Ok(( + ms.parse().map_err(|_| bad())?, + seq.parse().map_err(|_| bad())?, + )) +} + +impl Keys { + fn get(&self, field: impl AsRef<[u8]>) -> Option<&[u8]> { + self.meta.get(field.as_ref()).map(Vec::as_slice) + } + + /// A field that holds text, such as `newest`. `None` when it's absent or not text. + fn text(&self, field: impl AsRef<[u8]>) -> Option<&str> { + self.get(field) + .and_then(|value| std::str::from_utf8(value).ok()) + } + + /// A field that holds a number, zero when it's absent. + fn number(&self, field: &str) -> i64 { + self.text(field) + .and_then(|number| number.parse().ok()) + .unwrap_or(0) + } + + fn set(&mut self, field: impl Into>, value: impl Into>) { + self.meta.insert(field.into(), value.into()); + } + + fn remove(&mut self, field: impl AsRef<[u8]>) { + self.meta.remove(field.as_ref()); + } + + /// The `check_live` step, which checks the create key, the format version and the expiry. + fn check_live(&self, now: i64) -> StreamResult<()> { + if self.get("v").is_none() { + return Err(refuse(Reason::NotFound, "nobody created the Stream")); + } + let v = self.number("v"); + if v > i64::from(FORMAT_VERSION) { + return Err(refuse( + Reason::FormatTooNew, + &format!("format {v} is newer than {FORMAT_VERSION}"), + )); + } + if self + .text("exp") + .and_then(|exp| exp.parse::().ok()) + .is_some_and(|exp| exp <= now) + { + return Err(refuse(Reason::Expired, "the Stream expired")); + } + Ok(()) + } + + fn refresh(&mut self, now: i64, last_write_ttl: Option, tombstone: i64) { + // An open Stream keeps the last-write TTL after its last write, and with none it stays + // until its close. A closed one keeps the Retention. + let ttl = if self.get("closed").is_none() { + last_write_ttl + } else { + Some(self.number("ret")) + }; + let Some(ttl) = ttl else { + self.remove("exp"); + self.meta_expires = 0; + self.log_expires = 0; + return; + }; + let exp = now.saturating_add(ttl); + self.set("exp", exp.to_string()); + self.meta_expires = exp.saturating_add(tombstone); + if self.log.is_some() { + self.log_expires = exp; + } + } + + /// The newest id the Stream gave. Redis drops it with an expired log, so the meta's `newest` + /// keeps the next ids above the old ones. + fn newest_id(&self) -> (u64, u64) { + let newest = self.text("newest").and_then(parse_id); + self.last_id.max(newest.unwrap_or_default()) + } + + /// Raises the trimmed offset to `newest` when the log is gone, as `mark_log_gone` in `lib.lua`. + /// The log took every Record with it, so a read from an older offset must fail. + fn mark_log_gone(&mut self) { + if self.log.is_some() { + return; + } + if let Some(newest) = self.text("newest").and_then(parse_id) { + self.raise_trimmed(newest); + } + } + + /// Sets the trimmed offset to `id`, unless it is already at or past it. + fn raise_trimmed(&mut self, id: (u64, u64)) { + let moves = self + .text("trimmed") + .and_then(parse_id) + .is_none_or(|trimmed| trimmed < id); + if moves { + self.set("trimmed", id_text(id)); + } + } + + /// Refuses `count` entries whose ids would not fit an offset. The ids go on from the newest id + /// the Stream gave, as `check_ids_fit` in `lib.lua`. + fn check_ids_fit(&self, now: i64, count: u64) -> StreamResult<()> { + let (ms, seq) = self.newest_id(); + let first = if now > ms as i64 { 0 } else { seq + 1 }; + if first + count > SEQ_LIMIT { + return Err(refuse( + Reason::StorageUnavailable, + "the entry ids of this millisecond would not fit an offset", + )); + } + Ok(()) + } + + /// Adds an entry as `add` in `lib.lua` does, with the next id after the newest one, in this + /// millisecond or a later one. + fn add(&mut self, now: i64, fields: Fields) -> String { + let now = now.max(0) as u64; + let (last_ms, last_seq) = self.newest_id(); + let id = if now > last_ms { + (now, 0) + } else { + (last_ms, last_seq + 1) + }; + self.last_id = id; + self.log + .get_or_insert_with(BTreeMap::new) + .insert(id, fields); + id_text(id) + } +} + +impl MemoryStore { + /// An empty Store. + pub fn new(retention: Duration, last_write_ttl: Option) -> Self { + Self { + prefix: crate::ids::DEFAULT_KEY_PREFIX.to_string(), + retention, + last_write_ttl, + streams: Mutex::default(), + appended: Notify::new(), + } + } + + fn key(&self, stream_id: &str) -> String { + StreamKeys::new(&self.prefix, stream_id).meta() + } + + /// Runs `step` on one Stream's keys after it drops the expired keys, as Redis does. + fn with(&self, stream_id: &str, step: impl FnOnce(&mut Keys, i64) -> T) -> T { + let now = now_ms(); + let mut streams = self.streams.lock(); + let key = self.key(stream_id); + let keys = streams.entry(key.clone()).or_default(); + if keys.meta_expires > 0 && keys.meta_expires <= now { + keys.meta.clear(); + keys.meta_expires = 0; + } + if keys.log_expires > 0 && keys.log_expires <= now { + keys.log = None; + keys.log_expires = 0; + keys.last_id = (0, 0); + } + let result = step(keys, now); + // Once every key is gone, Redis has no state of the Stream, not even its newest id. + if keys.meta.is_empty() && keys.log.is_none() { + streams.remove(&key); + } + result + } +} + +#[cfg(any(test, feature = "test-support"))] +impl MemoryStore { + /// Changes one meta field, for tests of what the scripts refuse. + #[doc(hidden)] + pub fn test_set_meta(&self, stream_id: &str, field: &[u8], value: &[u8]) { + self.with(stream_id, |keys, _| keys.set(field, value)); + } + + /// One meta field, for tests. + #[doc(hidden)] + pub fn test_meta(&self, stream_id: &str, field: &[u8]) -> Option> { + self.with(stream_id, |keys, _| keys.get(field).map(<[u8]>::to_vec)) + } + + /// The ms left on the log and on the meta, -1 for a key with no expiry and -2 for a key that + /// doesn't exist, as `PTTL`. + #[doc(hidden)] + pub fn test_ttls(&self, stream_id: &str) -> (i64, i64) { + self.with(stream_id, |keys, now| { + let ttl = |expires: i64| if expires > 0 { expires - now } else { -1 }; + let log = if keys.log.is_some() { + ttl(keys.log_expires) + } else { + -2 + }; + let meta = if keys.meta.is_empty() { + -2 + } else { + ttl(keys.meta_expires) + }; + (log, meta) + }) + } + + /// Adds an entry with an explicit id, as `XADD `, for tests. + #[doc(hidden)] + pub fn test_add_entry(&self, stream_id: &str, id: &str) { + self.with(stream_id, |keys, _| { + let id = parse_id(id).expect("an entry id"); + keys.last_id = keys.last_id.max(id); + keys.log + .get_or_insert_with(BTreeMap::new) + .insert(id, Fields::default()); + }); + } + + /// The number of log entries, as `XLEN`, for tests. + #[doc(hidden)] + pub fn test_log_len(&self, stream_id: &str) -> usize { + self.with(stream_id, |keys, _| { + keys.log.as_ref().map_or(0, BTreeMap::len) + }) + } + + /// Deletes the log, as `DEL`, for tests. + #[doc(hidden)] + pub fn test_drop_log(&self, stream_id: &str) { + self.with(stream_id, |keys, _| { + keys.log = None; + keys.log_expires = 0; + keys.last_id = (0, 0); + }); + } +} + +fn offsets(first: &str, last: &str) -> StreamResult> { + if first.is_empty() || last.is_empty() { + return Ok(None); + } + Ok(Some(( + offset_of_entry_id(first)?, + offset_of_entry_id(last)?, + ))) +} + +#[async_trait::async_trait] +impl Store for MemoryStore { + fn capabilities(&self) -> Capabilities { + Capabilities::ALL + } + + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult { + let stream_id = stream_ref.id.as_str(); + let retention = create.retention.filter(|r| !r.is_zero()); + let retention = millis(retention.unwrap_or(self.retention)); + let request_id = create.request_id; + let (last_write_ttl, tombstone) = (self.last_write_ttl.map(millis), millis(TOMBSTONE)); + let created = self.with(stream_id, |keys, now| { + if keys.get("ret").is_none() { + keys.set("v", FORMAT_VERSION.to_string()); + keys.set("ret", retention.to_string()); + if !request_id.is_empty() { + keys.set("rid", request_id); + } + keys.refresh(now, last_write_ttl, tombstone); + return Ok((CreateOutcome::Created, retention)); + } + keys.check_live(now)?; + if keys.get("closed").is_some() { + return Err(refuse(Reason::Closed, "the Stream is closed")); + } + // A retry of the create that made the Stream gets its answer again. + if !request_id.is_empty() && keys.get("rid") == Some(request_id.as_bytes()) { + return Ok((CreateOutcome::Created, keys.number("ret"))); + } + if create.conflict == ConflictPolicy::Fail { + return Err(refuse(Reason::AlreadyExists, "the Stream exists")); + } + Ok((CreateOutcome::Exists, keys.number("ret"))) + }); + let (outcome, stored) = created?; + Ok(Created { + outcome, + retention: Duration::from_millis(stored.max(0) as u64), + }) + } + + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult { + let stream_id = stream_ref.id.as_str(); + let producer = batch.producer; + let kind = ProducerKind::try_from(producer)?; + let epoch = epoch_key(producer.kind.as_ref()); + let (last_write_ttl, tombstone) = (self.last_write_ttl.map(millis), millis(TOMBSTONE)); + let field = format!("p|{}|{}", kind as i32, producer.id); + let appended = self.with(stream_id, |keys, now| { + // The format version comes first, since a newer format can change the Producer field. + keys.check_live(now)?; + let stored = keys.get(&field).map(ProducerField::try_from).transpose()?; + // Every append sets the fence to its epoch, so the fence is the last batch's epoch. + let previous = stored.filter(|stored| stored.fence == epoch); + // The lock also covers the dedupe check, so no other batch takes the same number. + let sequence = match batch.sequence { + 0 => previous.map_or(1, |previous| previous.sequence + 1), + sequence => sequence, + }; + if !(1..=MAX_SEQUENCE).contains(&sequence) { + return Err(refuse( + Reason::Invalid, + "the sequence must be from 1 to 2^53 - 1", + )); + } + if let Some(previous) = previous { + if sequence == previous.sequence { + if previous.digest != batch.digest { + return Err(refuse( + Reason::SeqDivergent, + &format!( + "sequence {sequence} has other content than the batch the Store \ + has" + ), + )); + } + let (first, last) = previous + .entries + .map(|((ms, seq), (last_ms, last_seq))| { + (format!("{ms}-{seq}"), format!("{last_ms}-{last_seq}")) + }) + .unwrap_or_default(); + return Ok((true, first, last)); + } + if sequence < previous.sequence { + return Err(refuse( + Reason::SequenceTooLow, + &format!( + "sequence {sequence} is below the last one, {}", + previous.sequence + ), + )); + } + } + if keys.get("closed").is_some() { + return Err(refuse(Reason::Closed, "the Stream is closed")); + } + if stored.is_some_and(|stored| epoch < stored.fence) { + return Err(refuse( + Reason::Fenced, + "the epoch is lower than the newest one of the Producer", + )); + } + let records = batch.records.len() as u64; + keys.check_ids_fit(now, records)?; + + keys.mark_log_gone(); + let (mut first, mut last) = (String::new(), String::new()); + for record in batch.records { + last = keys.add( + now, + Fields { + record: Some(record.clone()), + }, + ); + if first.is_empty() { + first = last.clone(); + } + } + let entries = if last.is_empty() { + None + } else { + Some((entry_id(&first)?, entry_id(&last)?)) + }; + let value = ProducerField { + fence: epoch, + sequence, + entries, + digest: batch.digest, + }; + keys.set(field.as_str(), value.encode()); + if !last.is_empty() { + keys.set("newest", last.clone()); + } + keys.refresh(now, last_write_ttl, tombstone); + Ok((false, first, last)) + }); + let (resend, first, last) = appended?; + if !resend && !last.is_empty() { + self.appended.notify_waiters(); + } + Ok(Appended { + offsets: offsets(&first, &last)?, + resend, + }) + } + + async fn close(&self, stream_ref: &StreamRef, producer: &Producer) -> StreamResult { + let stream_id = stream_ref.id.as_str(); + let kind = ProducerKind::try_from(producer)?; + let epoch = epoch_key(producer.kind.as_ref()); + let field = format!("p|{}|{}", kind as i32, producer.id); + let tombstone = millis(TOMBSTONE); + let closed = self.with(stream_id, |keys, now| { + keys.check_live(now)?; + if keys.get("closed").is_some() { + return Ok(true); + } + let stored = keys.get(&field).map(ProducerField::try_from).transpose()?; + if stored.is_some_and(|stored| epoch < stored.fence) { + return Err(refuse( + Reason::Fenced, + "the epoch is lower than the newest one of the Producer", + )); + } + keys.set("closed", "1"); + keys.refresh(now, None, tombstone); + Ok(false) + }); + let already = closed?; + // No entry wakes a waiting read, so the close wakes it. + if !already { + self.appended.notify_waiters(); + } + Ok(Closed { already }) + } + + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)> { + let stream_id = stream_ref.id.as_str(); + let start = if from > 0 { + parse_id(&entry_id_of_offset(from)).unwrap_or((0, 0)) + } else { + (0, 0) + }; + self.with(stream_id, |keys, now| { + let offset = |field: &str| -> StreamResult { + keys.text(field).map_or(Ok(0), offset_of_entry_id) + }; + let number = |field: &str| keys.number(field); + let state = StreamState { + exists: keys.get("v").is_some(), + version: number("v") as u32, + closed: keys.get("closed").is_some(), + newest: offset("newest")?, + trimmed: offset("trimmed")?, + expires_ms: number("exp"), + now_ms: now, + }; + let mut entries = vec![]; + if let Some(log) = keys.log.as_ref() { + for (id, fields) in log.range(start..).take(count) { + entries.push(Entry { + offset: offset_of_entry_id(&id_text(*id))?, + record: fields.record.clone(), + }); + } + } + Ok((state, entries)) + }) + } + + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult { + let stream_id = stream_ref.id.as_str(); + let after = if after > 0 { + parse_id(&entry_id_of_offset(after)).unwrap_or((0, 0)) + } else { + (0, 0) + }; + loop { + // Listen before the check, so an append between the two still wakes the wait. + let appended = self.appended.notified(); + let found = self.with(stream_id, |keys, _| { + keys.get("closed").is_some() + || keys + .log + .as_ref() + .and_then(|log| log.keys().next_back()) + .is_some_and(|id| *id > after) + }); + if found { + return Ok(true); + } + if tokio::time::timeout_at(deadline, appended).await.is_err() { + return Ok(false); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_id_follows_the_newest_one_within_a_millisecond() { + let mut keys = Keys::default(); + assert_eq!(keys.add(5, Fields::default()), "5-0"); + assert_eq!(keys.add(5, Fields::default()), "5-1"); + // The ids keep increasing when the clock steps back. + assert_eq!(keys.add(3, Fields::default()), "5-2"); + assert_eq!(keys.add(9, Fields::default()), "9-0"); + } + + #[test] + fn a_producer_field_matches_the_shared_vectors() { + let vectors: serde_json::Value = + serde_json::from_str(include_str!("../testdata/fields.json")).unwrap(); + let hex = |text: &str| -> Vec { + (0..text.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&text[i..i + 2], 16).unwrap()) + .collect() + }; + for case in vectors["producer_values"].as_array().unwrap() { + let text = |name: &str| case[name].as_str().unwrap(); + let key = |name: &str| case[name].as_str().map(|key| hex(key).try_into().unwrap()); + let entries = (!text("first").is_empty()).then(|| { + ( + entry_id(text("first")).unwrap(), + entry_id(text("last")).unwrap(), + ) + }); + let field = ProducerField { + fence: key("fence").unwrap(), + sequence: case["sequence"].as_i64().unwrap(), + entries, + digest: hex(text("digest")).try_into().unwrap(), + }; + let value = hex(text("value")); + assert_eq!(field.encode(), value, "{}", text("name")); + assert_eq!(ProducerField::try_from(value.as_slice()).unwrap(), field); + } + } +} diff --git a/crates/streams/src/store.rs b/crates/streams/src/store.rs new file mode 100644 index 000000000..3b9638428 --- /dev/null +++ b/crates/streams/src/store.rs @@ -0,0 +1,285 @@ +//! The calls that every Store gives the Stream service. The Redis Store of +//! `temporalio-streams-redis` is for production, and the memory Store is for tests. Both apply the +//! same rules. + +use crate::{ + Reason, StreamError, StreamResult, + proto::{Producer, StreamRef}, +}; +use std::time::Duration; +use tokio::time::Instant; + +mod lazy; + +pub use lazy::LazyStore; + +/// The Retention of a Stream whose create sets none, when a Store's options set no other. +pub const DEFAULT_RETENTION: Duration = Duration::from_secs(7 * 24 * 60 * 60); + +/// The format version this crate writes. A Stream with a higher one refuses writes. +pub const FORMAT_VERSION: u32 = 1; +/// The maximum size of a stored Record, in bytes. +pub const MAX_RECORD_BYTES: usize = 1536 * 1024; +/// The maximum size of a stored append, in bytes. +pub const MAX_BATCH_BYTES: usize = 4 * 1024 * 1024; +/// The maximum number of Records in an append. Many small Records fit under [MAX_BATCH_BYTES], but +/// they keep a Store busy for long. +pub const MAX_BATCH_RECORDS: usize = 1000; +/// The highest batch `sequence`. A Lua number is exact for integers only up to this value. +pub const MAX_SEQUENCE: i64 = (1 << 53) - 1; +/// How long the meta key stays after the log key expires, so that a read can tell an expired Stream +/// from a missing one. +pub const TOMBSTONE: Duration = Duration::from_secs(30 * 24 * 60 * 60); +/// The longest Retention or last-write TTL, which is the largest proto Duration. Its ms plus the +/// time of a write stay exact in a Lua number. +pub const MAX_TTL: Duration = Duration::from_secs(315_576_000_000); + +/// What a create does when its Stream exists and is open, and the create isn't a retry of the +/// create that made the Stream. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum ConflictPolicy { + /// The create fails with [Reason::AlreadyExists]. + #[default] + Fail, + /// The create gets the open Stream, which keeps its own Retention. + UseExisting, +} + +/// A create of a Stream. +#[derive(Debug, Clone, Default)] +pub struct Create<'a> { + /// The Retention, `None` for the Store default. The service always sends `None`, since the + /// Store config holds the Retention. A test may set a short one. + pub retention: Option, + /// What the create does when the Stream exists. + pub conflict: ConflictPolicy, + /// Unique for each create call, and the same on its retries. Empty for none. The Store keeps + /// the request id of the create that made the Stream, so a retry after a lost answer succeeds. + pub request_id: &'a str, +} + +/// What a create found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CreateOutcome { + /// The create wrote the key, or it is a retry of the create that wrote it. + Created, + /// The Stream exists and is open, and the create uses it. The create doesn't change its + /// Retention. + Exists, +} + +/// The result of a create. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Created { + /// What the create found. + pub outcome: CreateOutcome, + /// The Retention of the Stream, the stored one if it existed. + pub retention: Duration, +} + +/// One batch of one Producer. +#[derive(Debug, Clone)] +pub struct Batch<'a> { + /// The Producer, with its epoch. + pub producer: &'a Producer, + /// The batch number in this epoch. Zero takes the next one of the epoch, which needs + /// [Capabilities::NEXT_SEQUENCE]. Such a batch has no dedupe, so its retry writes it again. + pub sequence: i64, + /// The encoded Records. + pub records: &'a [Vec], + /// The [crate::digest::batch_digest] of the batch, which the dedupe entry keeps. + pub digest: [u8; crate::digest::DIGEST_LEN], +} + +/// The result of an append. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Appended { + /// The offsets of the first and the last entry, `None` for an empty batch. + pub offsets: Option<(i64, i64)>, + /// The Store already had the batch, so the offsets are those of the first result. + pub resend: bool, +} + +/// The result of a close. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Closed { + /// The Stream was already closed. + pub already: bool, +} + +/// The state of a Stream as one read saw it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct StreamState { + /// The meta key exists, live or as a Tombstone. + pub exists: bool, + /// The format version. + pub version: u32, + /// The Stream is closed. Every later append fails, so `newest` is final. + pub closed: bool, + /// The offset of the newest entry written, zero if none. + pub newest: i64, + /// The newest offset when a write found the log gone, zero if none. + pub trimmed: i64, + /// When the Stream stops taking writes, in ms since 1970. Zero for an open Stream with no + /// last-write TTL, which stays until its close. + pub expires_ms: i64, + /// The server time of the read, in ms since 1970. + pub now_ms: i64, +} + +impl StreamState { + /// The Stream is a Tombstone. + pub fn expired(&self) -> bool { + self.exists && self.expires_ms > 0 && self.expires_ms <= self.now_ms + } + + /// The offset below which the log lost the Records. The Retention counts from the close, so + /// it hides no Record of a Stream that hasn't expired. + pub fn retained_from(&self) -> i64 { + self.trimmed + 1 + } +} + +/// One log entry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + /// Its offset. + pub offset: i64, + /// The encoded Record. + pub record: Option>, +} + +/// The rules that a Store applies atomically beyond the log. The service checks a capability before +/// a call that needs it. So a Store that lacks the capability refuses the call, and doesn't give +/// weaker guarantees without notice. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Capabilities(u8); + +impl Capabilities { + /// Each append checks the Producer's epoch and its last `sequence` before it writes. The Store + /// keeps both for as long as the Stream exists, also for a Producer with no recent append. + pub const PRODUCER_FENCE: Self = Self(1); + /// A close marks the Stream closed and wakes the waiting reads. Every later append is + /// refused. + pub const CLOSE: Self = Self(1 << 1); + /// An append with sequence zero takes the next one of the Producer's epoch, in the same atomic + /// step as the dedupe check. So two such batches never get the same number. + pub const NEXT_SEQUENCE: Self = Self(1 << 2); + /// Every capability. + pub const ALL: Self = Self((1 << 3) - 1); + + const NAMES: [(Self, &'static str); 3] = [ + (Self::PRODUCER_FENCE, "producer_fence"), + (Self::CLOSE, "close"), + (Self::NEXT_SEQUENCE, "next_sequence"), + ]; + + /// Both sets. + pub const fn union(self, other: Self) -> Self { + Self(self.0 | other.0) + } + + /// This set without the other one. + pub const fn without(self, other: Self) -> Self { + Self(self.0 & !other.0) + } + + /// Whether every capability of `other` is in this set. + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } + + /// The names of the capabilities in this set. + pub fn names(self) -> Vec<&'static str> { + Self::NAMES + .iter() + .filter(|(capability, _)| self.contains(*capability)) + .map(|(_, name)| *name) + .collect() + } + + /// Refuses a call that needs a capability this set lacks, and names the missing ones. + pub fn require(self, needed: Self) -> StreamResult<()> { + let missing = needed.without(self); + if missing == Self::default() { + return Ok(()); + } + Err(unsupported(&missing.names().join(", "))) + } +} + +fn unsupported(capability: &str) -> StreamError { + StreamError::new( + Reason::Unsupported, + format!("The Store lacks the capability {capability}"), + ) +} + +/// The calls of a Store. Each one is atomic, as one Lua script is on Redis. +/// +/// The required calls make the log contract. They are a create key, an atomic batch append, an +/// ordered read with the Stream's state, stable offsets, a Retention that counts from the close and +/// a wait for the next entry. An append applies the rules of the capabilities that the Store +/// declares. The optional calls fail by default. +#[async_trait::async_trait] +pub trait Store: Send + Sync + 'static { + /// What the Store does atomically beyond the log. It has no default, so a Store that wraps + /// another one must say what it forwards. + fn capabilities(&self) -> Capabilities; + + /// Creates a Stream. + /// + /// The create of an existing Stream fails with [Reason::Expired] or [Reason::Closed], in this + /// order. Then a retry of the create that made the Stream, with the same request id, gets + /// [CreateOutcome::Created] and changes nothing. Any other create applies its [ConflictPolicy]. + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult; + + /// Appends the whole batch, or nothing of it. The batch's Records get consecutive offsets that + /// end at the last offset of [Appended::offsets], so a caller knows the offset of each one. + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult; + + /// Reads the Stream's state and up to `count` entries from `from`, inclusive. Zero reads + /// from the start of the log. + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)>; + + /// Waits until the log has an entry after `after`, the Stream is closed, or `deadline` passes. + /// Returns whether an entry came or the Stream is closed. It may return `false` before the + /// deadline, and then the caller reads again. + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult; + + /// Closes a Stream. A close writes no log entry, and a close of a closed Stream changes + /// nothing. Needs [Capabilities::CLOSE]. + async fn close(&self, _stream_ref: &StreamRef, _producer: &Producer) -> StreamResult { + Err(unsupported("close")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_missing_capability_is_refused_by_name() { + let some = Capabilities::PRODUCER_FENCE.union(Capabilities::CLOSE); + some.require(Capabilities::CLOSE).unwrap(); + let error = some + .require(Capabilities::CLOSE.union(Capabilities::NEXT_SEQUENCE)) + .unwrap_err(); + assert_eq!(error.reason, Reason::Unsupported); + assert!( + error.message.ends_with("next_sequence"), + "{}", + error.message + ); + } +} diff --git a/crates/streams/src/store/lazy.rs b/crates/streams/src/store/lazy.rs new file mode 100644 index 000000000..67771b7e4 --- /dev/null +++ b/crates/streams/src/store/lazy.rs @@ -0,0 +1,181 @@ +//! A Store that connects on its first call. + +use crate::{ + StreamError, StreamResult, + proto::{Producer, StreamRef}, + store::{Appended, Batch, Capabilities, Closed, Create, Created, Entry, Store, StreamState}, +}; +use std::{future::Future, pin::Pin}; +use tokio::{sync::OnceCell, time::Instant}; + +type Connect = + Box Pin> + Send>> + Send + Sync>; + +/// A Store that connects its inner Store on the first call, so a bridge can build the runtime with +/// no async runtime, and a Workflow that never publishes never connects. A failed connect is tried +/// again on the next call. An inner Store that lacks a declared capability is refused once, and +/// every later call fails with that refusal. +pub struct LazyStore { + capabilities: Capabilities, + connect: Connect, + store: OnceCell>, +} + +impl std::fmt::Debug for LazyStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LazyStore") + .field("capabilities", &self.capabilities) + .field("connected", &matches!(self.store.get(), Some(Ok(_)))) + .finish_non_exhaustive() + } +} + +impl LazyStore { + /// A Store that `connect` connects on its first call. The runtime asks for the capabilities + /// before any call connects, so the Store declares `capabilities` when it is built, and a + /// connect fails if the inner Store lacks one of them. + pub fn new(capabilities: Capabilities, connect: F) -> Self + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + E: Into, + { + Self { + capabilities, + connect: Box::new(move || { + let connecting = connect(); + Box::pin(async move { connecting.await.map_err(Into::into) }) + }), + store: OnceCell::new(), + } + } + + async fn store(&self) -> StreamResult<&S> { + let connected = self + .store + .get_or_try_init(|| async { + let store = (self.connect)().await?; + // The same Store lacks the capability on the next connect too. + Ok::<_, StreamError>( + store + .capabilities() + .require(self.capabilities) + .map(|()| store), + ) + }) + .await?; + connected.as_ref().map_err(Clone::clone) + } +} + +#[async_trait::async_trait] +impl Store for LazyStore { + fn capabilities(&self) -> Capabilities { + self.capabilities + } + + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult { + self.store().await?.create(stream_ref, create).await + } + + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult { + self.store().await?.append(stream_ref, batch).await + } + + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)> { + self.store().await?.read(stream_ref, from, count).await + } + + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult { + self.store() + .await? + .wait_after(stream_ref, after, deadline) + .await + } + + async fn close(&self, stream_ref: &StreamRef, producer: &Producer) -> StreamResult { + self.store().await?.close(stream_ref, producer).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Reason, memory::MemoryStore, store::DEFAULT_RETENTION}; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + fn stream() -> StreamRef { + StreamRef { + id: "s".to_string(), + ..Default::default() + } + } + + fn memory() -> MemoryStore { + MemoryStore::new(DEFAULT_RETENTION, None) + } + + #[tokio::test] + async fn a_failed_connect_is_tried_again_on_the_next_call() { + let connects = Arc::new(AtomicUsize::new(0)); + let counted = connects.clone(); + let lazy = LazyStore::new(Capabilities::ALL, move || { + let first = counted.fetch_add(1, Ordering::SeqCst) == 0; + async move { + if first { + return Err(StreamError::storage_unavailable("the Store is down")); + } + Ok(memory()) + } + }); + assert_eq!(lazy.capabilities(), Capabilities::ALL); + assert_eq!(connects.load(Ordering::SeqCst), 0); + let error = lazy + .create(&stream(), &Create::default()) + .await + .unwrap_err(); + assert_eq!(error.reason, Reason::StorageUnavailable); + lazy.create(&stream(), &Create::default()).await.unwrap(); + let (state, _) = lazy.read(&stream(), 0, 0).await.unwrap(); + assert!(state.exists); + assert_eq!(connects.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn a_connect_refuses_a_store_that_lacks_a_declared_capability_once() { + let connects = Arc::new(AtomicUsize::new(0)); + let counted = connects.clone(); + let lazy = LazyStore::new(Capabilities::ALL, move || { + counted.fetch_add(1, Ordering::SeqCst); + async { + let fewer = LazyStore::new(Capabilities::CLOSE, || async { + Ok::<_, StreamError>(memory()) + }); + Ok::<_, StreamError>(fewer) + } + }); + for _ in 0..2 { + let error = lazy.read(&stream(), 0, 0).await.unwrap_err(); + assert_eq!(error.reason, Reason::Unsupported); + assert!( + error.message.contains("producer_fence"), + "{}", + error.message + ); + } + assert_eq!(connects.load(Ordering::SeqCst), 1); + } +} diff --git a/crates/streams/src/testing/fields.rs b/crates/streams/src/testing/fields.rs new file mode 100644 index 000000000..75fd320ea --- /dev/null +++ b/crates/streams/src/testing/fields.rs @@ -0,0 +1,32 @@ +//! The fields of the `:meta` hash. A Producer field is `p||`. The id is opaque +//! and ends the name, so a `|` in it makes no other field. + +/// The format version of the Stream. +pub const VERSION: &str = "v"; + +/// The field of a Producer. Its value is 88 bytes, with big-endian numbers: +/// +/// - 40, the fence, the Producer's newest epoch key. It is also the epoch of the last batch. +/// - 8, the last batch's sequence. +/// - 12 and 12, the last batch's first and last entry ids, each the ms in 8 bytes and the seq in +/// 4. They are zero when the batch wrote no entry. +/// - 16, the last batch's digest, which a resend must match. +/// +/// A Store refuses a value of another length. +pub fn producer(kind: i32, producer_id: &str) -> String { + format!("p|{kind}|{producer_id}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_fields_match_the_shared_vectors() { + let case: serde_json::Value = + serde_json::from_str(include_str!("../../testdata/fields.json")).unwrap(); + let text = |field: &str| case[field].as_str().unwrap(); + let kind = case["kind"].as_i64().unwrap() as i32; + assert_eq!(producer(kind, text("producer_id")), text("producer")); + } +} diff --git a/crates/streams/src/testing/mod.rs b/crates/streams/src/testing/mod.rs new file mode 100644 index 000000000..b60561ad3 --- /dev/null +++ b/crates/streams/src/testing/mod.rs @@ -0,0 +1,142 @@ +//! The shared suites of the Stream rules. Each Store crate runs them on its own Stores, so that +//! every Store keeps the same rules. This crate runs them on the memory Store. + +mod support; + +pub mod fields; +pub mod store; + +pub use support::Needs; + +use crate::{ + memory::MemoryStore, + store::{DEFAULT_RETENTION, Store}, +}; +use futures_util::{FutureExt, future::join_all}; +use std::{future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc}; + +/// The state under a Store's calls, which some cases read or change. +#[async_trait::async_trait] +pub trait RawKeys: Send + Sync { + /// Sets one meta field of a Stream. Field names and values are bytes, as Redis keeps them. + async fn set_meta(&self, stream: &str, field: &[u8], value: &[u8]); + /// One meta field of a Stream. + async fn meta(&self, stream: &str, field: &[u8]) -> Option>; + /// The ms left on the log and on the meta, as `PTTL` gives them. + async fn ttls(&self, stream: &str) -> (i64, i64); + /// Adds a log entry with an explicit id, as `XADD `. + async fn add_raw_entry(&self, stream: &str, id: &str); + /// The number of log entries. + async fn log_len(&self, stream: &str) -> usize; + /// Deletes the log, as an eviction would. + async fn drop_log(&self, stream: &str); +} + +/// A Store that the suites run on, with the server under it. +#[async_trait::async_trait] +pub trait TestStore: Send + Sync { + /// What the Store gives. A case that needs more doesn't run on it. + fn gives(&self) -> Needs; + /// A Store for one case, and the state under it. + async fn connect(&self) -> (Arc, Arc); +} + +/// The memory Store, with a new Store for each case. +#[derive(Debug, Default)] +pub struct MemoryTestStore; + +#[async_trait::async_trait] +impl TestStore for MemoryTestStore { + fn gives(&self) -> Needs { + Needs::ALL + } + + async fn connect(&self) -> (Arc, Arc) { + let store = Arc::new(MemoryStore::new(DEFAULT_RETENTION, None)); + (store.clone(), store) + } +} + +#[async_trait::async_trait] +impl RawKeys for MemoryStore { + async fn set_meta(&self, stream: &str, field: &[u8], value: &[u8]) { + self.test_set_meta(stream, field, value) + } + + async fn meta(&self, stream: &str, field: &[u8]) -> Option> { + self.test_meta(stream, field) + } + + async fn ttls(&self, stream: &str) -> (i64, i64) { + self.test_ttls(stream) + } + + async fn add_raw_entry(&self, stream: &str, id: &str) { + self.test_add_entry(stream, id) + } + + async fn log_len(&self, stream: &str) -> usize { + self.test_log_len(stream) + } + + async fn drop_log(&self, stream: &str) { + self.test_drop_log(stream) + } +} + +/// One case of a suite. +pub struct Case { + /// The name of the case, for the report. + pub name: &'static str, + /// Runs the case on each Store under test that gives what it needs. + pub run: fn() -> Pin>>, +} + +macro_rules! case { + ($name:ident) => { + $crate::testing::Case { + name: stringify!($name), + run: || Box::pin($name()), + } + }; +} +pub(crate) use case; + +tokio::task_local! { + static TEST_STORES: Vec>; +} + +/// The Stores of the running suite. +pub(crate) fn test_stores() -> Vec> { + TEST_STORES.with(Clone::clone) +} + +/// Runs `cases` on `stores`, all at once, and then fails with the names of the cases that +/// failed. A case that fails prints its panic as it goes. +pub async fn run(stores: Vec>, cases: &[Case]) { + let outcomes = TEST_STORES + .scope( + stores, + join_all(cases.iter().map(|case| async move { + let passed = AssertUnwindSafe((case.run)()).catch_unwind().await.is_ok(); + (case.name, passed) + })), + ) + .await; + let failed: Vec<_> = outcomes + .into_iter() + .filter(|(_, passed)| !passed) + .map(|(name, _)| name) + .collect(); + assert!(failed.is_empty(), "these cases failed: {failed:?}"); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn the_store_suite_passes_on_the_memory_store() { + run(vec![Arc::new(MemoryTestStore)], store::CASES).await; + } +} diff --git a/crates/streams/src/testing/store.rs b/crates/streams/src/testing/store.rs new file mode 100644 index 000000000..5aea896a4 --- /dev/null +++ b/crates/streams/src/testing/store.rs @@ -0,0 +1,743 @@ +//! The Store suite. Every Store runs it, so that each one keeps the same rules. +//! +//! A case says its guarantee and what it needs from a Store in its comment. + +use super::{ + case, fields, + support::{Needs, Run, Target, digest, record, records_of, stores_for}, +}; +use crate::{ + Reason, StreamResult, + epoch::{ProducerKind, epoch_key}, + memory::{EntryId, ProducerField}, + offset::{entry_id_of_offset, offset_of_entry_id}, + proto::{ActivityEpoch, ActivityProducer, ClientProducer, Producer, producer::Kind}, + store::{Batch, ConflictPolicy, Create, CreateOutcome, Created}, +}; +use futures_util::future::join_all; +use prost_types::Timestamp; +use std::time::Duration; + +fn activity(run: &str, attempt: i32, scheduled: i64) -> Producer { + Producer { + id: format!("act:{run}:a:s"), + kind: Some(Kind::WorkflowActivity(ActivityProducer { + run_id: run.to_string(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: Some(Timestamp { + seconds: scheduled, + nanos: 0, + }), + attempt, + }), + })), + } +} + +fn client() -> Producer { + Producer { + id: "client:me:p1".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } +} + +/// A second client Producer, which every Store accepts as sent. +fn other_client() -> Producer { + Producer { + id: "client:other:p2".to_string(), + ..client() + } +} + +async fn append( + target: &Target, + producer: &Producer, + sequence: i64, + texts: &[&str], +) -> Result { + let records: Vec<_> = texts.iter().map(|text| record(producer, text)).collect(); + target + .store + .append( + &target.stream_ref(), + &Batch { + producer, + sequence, + records: &records, + digest: digest(texts), + }, + ) + .await +} + +/// The create of a Stream that uses an existing one. +fn use_existing() -> Create<'static> { + Create { + conflict: ConflictPolicy::UseExisting, + ..Default::default() + } +} + +/// A create writes the key once. A create with another Retention finds the Stream and keeps the +/// stored Retention. A create with no request id keeps none. +async fn a_create_writes_the_key_once_and_keeps_its_retention() { + for target in stores_for(Needs::STORE_ANSWERS).await { + assert_eq!( + append(&target, &client(), 1, &["x"]) + .await + .unwrap_err() + .reason, + Reason::NotFound + ); + let day = Create { + retention: Some(Duration::from_secs(86_400)), + ..use_existing() + }; + let created = target.create_with(&day).await.unwrap(); + assert_eq!(created.outcome, CreateOutcome::Created); + let again = target.create_with(&day).await.unwrap(); + assert_eq!(again.outcome, CreateOutcome::Exists); + let other = target.create_with(&use_existing()).await.unwrap(); + assert_eq!(other.outcome, CreateOutcome::Exists); + assert_eq!(other.retention, Duration::from_secs(86_400)); + if target.gives.gives(Needs::RAW_KEYS) { + assert_eq!(target.meta("rid").await, None); + } + } +} + +/// A create of an open Stream fails as `already_exists`, unless it uses the Stream or it is a +/// retry of the create that made the Stream. Such a retry has the same request id, and it succeeds +/// as a create with either policy. The Store keeps that request id as `rid`. +async fn a_create_of_an_open_stream_fails_unless_it_uses_it_or_retries() { + for target in stores_for(Needs::CALLS).await { + let first = Create { + request_id: "create-1", + ..Default::default() + }; + let outcome = |created: StreamResult| created.unwrap().outcome; + let reason = |created: StreamResult| created.unwrap_err().reason; + assert_eq!( + outcome(target.create_with(&first).await), + CreateOutcome::Created + ); + let other = Create { + request_id: "create-2", + ..Default::default() + }; + for create in [&other, &Create::default()] { + assert_eq!( + reason(target.create_with(create).await), + Reason::AlreadyExists + ); + } + let used = Create { + request_id: "create-2", + ..use_existing() + }; + assert_eq!( + outcome(target.create_with(&used).await), + CreateOutcome::Exists + ); + let retry = Create { + request_id: "create-1", + ..use_existing() + }; + for create in [&first, &retry] { + assert_eq!( + outcome(target.create_with(create).await), + CreateOutcome::Created + ); + } + if target.gives.gives(Needs::RAW_KEYS) { + assert_eq!(target.meta("rid").await.as_deref(), Some(&b"create-1"[..])); + } + } +} + +/// A create of an expired or a closed Stream fails as `expired` or `closed`, in this order. These +/// come before the conflict policy and before a retry of the create that made the Stream. +async fn closed_and_expired_come_before_already_exists() { + for target in stores_for(Needs::RAW_KEYS).await { + let first = Create { + request_id: "create-1", + ..Default::default() + }; + target.create_with(&first).await.unwrap(); + let reason = |created: StreamResult| created.unwrap_err().reason; + target.close(&client()).await.unwrap(); + for create in [&first, &Create::default()] { + assert_eq!(reason(target.create_with(create).await), Reason::Closed); + } + target.set_meta("exp", "1").await; + assert_eq!(reason(target.create_with(&first).await), Reason::Expired); + } +} + +/// A resend with the last sequence gets the offsets of its first result, even after another +/// Producer wrote, and a lower sequence is refused. +async fn a_resend_gets_its_first_positions_and_a_lower_sequence_is_refused() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + let first = append(&target, &p, 1, &["p1", "p2"]).await.unwrap(); + // Another Producer writes between the batch and its resend. + append(&target, &other_client(), 1, &["q1"]).await.unwrap(); + let resend = append(&target, &p, 1, &["p1", "p2"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + if target.gives.gives(Needs::STORE_ANSWERS) { + assert!(!first.resend && resend.resend); + } + append(&target, &p, 2, &["p3"]).await.unwrap(); + assert_eq!( + append(&target, &p, 1, &["p1"]).await.unwrap_err().reason, + Reason::SequenceTooLow + ); + let texts = target.texts().await; + assert_eq!(texts, ["p1", "p2", "q1", "p3"]); + } +} + +/// A Producer id is bytes. An id with the field separator and non-ASCII text keeps its own dedupe +/// state, apart from an id that it starts with. +async fn a_producer_id_is_bytes_with_its_own_state() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let odd = Producer { + id: "ingest|eu:1 é".to_string(), + ..client() + }; + let plain = Producer { + id: "ingest".to_string(), + ..client() + }; + let first = append(&target, &odd, 1, &["a"]).await.unwrap(); + append(&target, &plain, 1, &["b"]).await.unwrap(); + let resend = append(&target, &odd, 1, &["a"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + assert_eq!(target.texts().await, ["a", "b"]); + } +} + +/// A resend with the last sequence and other content fails as `seq_divergent` and writes nothing. +async fn a_resend_with_other_content_fails_as_divergent() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + append(&target, &p, 1, &["p1"]).await.unwrap(); + let divergent = append(&target, &p, 1, &["other"]).await.unwrap_err(); + assert_eq!(divergent.reason, Reason::SeqDivergent); + assert_eq!(target.texts().await, ["p1"]); + } +} + +/// A lower epoch is fenced for each Producer kind that has an epoch. A client has none, so two +/// writes of one client Producer are never fenced. +async fn a_lower_epoch_is_fenced_for_each_producer_kind() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + let run = Run::new("r1"); + let la = run.local_activity("k", 8, 1, 2); + target.batch(&la, 1, &["la2"]).await.unwrap(); + let stale = target + .batch(&run.local_activity("k", 8, 1, 1), 1, &["la1"]) + .await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + append(&target, &activity("r1", 2, 20), 1, &["a2"]) + .await + .unwrap(); + let stale = append(&target, &activity("r1", 3, 15), 1, &["a3"]).await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + append(&target, &client(), 1, &["c1"]).await.unwrap(); + append(&target, &client(), 2, &["c2"]).await.unwrap(); + assert_eq!(target.texts().await, ["la2", "a2", "c1", "c2"]); + } +} + +/// A close refuses later appends but not a resend of an earlier batch, and a create of the Stream +/// fails as closed. With `STORE_ANSWERS`, a second close returns `already`. +async fn a_close_stops_appends_but_not_a_resend() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + let first = append(&target, &p, 1, &["a"]).await.unwrap(); + let closed = target.close(&p).await.unwrap(); + assert_eq!( + append(&target, &p, 2, &["b"]).await.unwrap_err().reason, + Reason::Closed + ); + assert_eq!( + append(&target, &p, 1, &["a"]).await.unwrap().offsets, + first.offsets + ); + let again = target.close(&p).await.unwrap(); + if target.gives.gives(Needs::STORE_ANSWERS) { + assert!(!closed.already && again.already); + } + let created = target.create_with(&use_existing()).await; + assert_eq!(created.unwrap_err().reason, Reason::Closed); + } +} + +/// A Stream with a newer format version refuses every write. A write checks the version before it +/// reads the Producer's `p|` field, since a newer format can change that field. +async fn a_newer_format_refuses_every_write() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let p = client(); + append(&target, &p, 1, &["a"]).await.unwrap(); + target.set_meta(fields::VERSION, "2").await; + let field = fields::producer(ProducerKind::Client as i32, &p.id); + target.set_meta(&field, "a field of a newer format").await; + let error = append(&target, &p, 2, &["b"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::FormatTooNew); + assert_eq!( + target.close(&p).await.unwrap_err().reason, + Reason::FormatTooNew + ); + let create = target.create_with(&use_existing()).await; + assert_eq!(create.unwrap_err().reason, Reason::FormatTooNew); + } +} + +/// An expired Stream refuses writes and its create. +async fn an_expired_stream_refuses_writes_and_its_create() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + target.set_meta("exp", "1").await; + let error = append(&target, &client(), 1, &["a"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::Expired); + let created = target.create_with(&use_existing()).await; + assert_eq!(created.unwrap_err().reason, Reason::Expired); + } +} + +/// With no last-write TTL, an open Stream's meta and log never expire, and the meta has no `exp`. +/// Its close gives both the Retention, and the meta the Tombstone after it. +async fn an_open_stream_with_no_last_write_ttl_never_expires_until_the_close() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + append(&target, &client(), 1, &["a"]).await.unwrap(); + let week = Duration::from_secs(7 * 86_400).as_millis() as i64; + assert_eq!(target.ttls().await, (-1, -1)); + assert_eq!(target.meta("exp").await, None); + target.close(&client()).await.unwrap(); + let (ttl, meta) = target.ttls().await; + assert!(0 < ttl && ttl <= week, "{ttl}"); + assert!( + meta > Duration::from_secs(36 * 86_400).as_millis() as i64, + "{meta}" + ); + } +} + +/// An open Stream keeps every Record past its Retention, while it is idle and after its next +/// append. No append trims a Record and no read hides one, since the Retention counts from the +/// close. +async fn an_open_stream_keeps_its_records_past_its_retention() { + for target in stores_for(Needs::RAW_KEYS.and(Needs::SHORT_TIMES)).await { + let retention = Duration::from_millis(300); + target + .create_with(&Create { + retention: Some(retention), + ..Default::default() + }) + .await + .unwrap(); + let old = append(&target, &client(), 1, &["old"]).await.unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + assert_eq!(target.texts().await, ["old"]); + append(&target, &client(), 2, &["new"]).await.unwrap(); + let (state, entries) = target + .store + .read(&target.stream_ref(), 0, 10) + .await + .unwrap(); + assert_eq!(state.trimmed, 0); + assert!(state.retained_from() <= old.offsets.unwrap().0); + assert_eq!(records_of(&entries), ["old", "new"]); + // A Store with a last-write TTL gives the log that TTL, and one with none gives no TTL. + let (ttl, _) = target.ttls().await; + assert!(ttl == -1 || ttl > retention.as_millis() as i64, "{ttl}"); + } +} + +/// A write that finds the log gone, as after an eviction, raises the trimmed offset to the meta's +/// newest one. Its offsets go above it, also with the clock behind it. +async fn offsets_go_on_after_the_log_is_gone() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + append(&target, &client(), 1, &["old"]).await.unwrap(); + // A newest id one minute ahead, as after the clock steps back. + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + let ahead = format!("{}-3", now + 60_000); + target.set_meta("newest", &ahead).await; + target.drop_log().await; + assert_eq!(target.ttls().await.0, -2); + let new = append(&target, &client(), 2, &["new"]).await.unwrap(); + let ahead = offset_of_entry_id(&ahead).unwrap(); + assert!(new.offsets.unwrap().0 > ahead); + let (state, _) = target.store.read(&target.stream_ref(), 0, 0).await.unwrap(); + assert_eq!(state.trimmed, ahead); + assert_eq!(target.texts().await, ["new"]); + } +} + +/// A wait for the next entry wakes on an append and ends at its deadline. +async fn a_blocked_read_wakes_on_an_append_and_ends_at_its_deadline() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let first = append(&target, &client(), 1, &["a"]).await.unwrap(); + let newest = first.offsets.unwrap().1; + let deadline = tokio::time::Instant::now() + Duration::from_millis(200); + let woke = target + .store + .wait_after(&target.stream_ref(), newest, deadline) + .await + .unwrap(); + assert!(!woke); + let store = target.store.clone(); + let stream = target.stream_ref(); + let waiter = tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + store.wait_after(&stream, newest, deadline).await + }); + tokio::time::sleep(Duration::from_millis(100)).await; + append(&target, &client(), 2, &["b"]).await.unwrap(); + assert!(waiter.await.unwrap().unwrap()); + } +} + +/// A close writes no log entry. It ends a wait for the next entry, and a wait on a closed Stream +/// ends at once, so a waiting read sees the close. +async fn a_close_writes_no_entry_and_ends_a_wait() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let first = append(&target, &client(), 1, &["a"]).await.unwrap(); + let newest = first.offsets.unwrap().1; + let store = target.store.clone(); + let stream = target.stream_ref(); + let waiter = tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + store.wait_after(&stream, newest, deadline).await + }); + tokio::time::sleep(Duration::from_millis(100)).await; + target.close(&client()).await.unwrap(); + assert!(waiter.await.unwrap().unwrap()); + let started = tokio::time::Instant::now(); + let deadline = started + Duration::from_secs(5); + let woke = target + .store + .wait_after(&target.stream_ref(), newest, deadline) + .await + .unwrap(); + assert!(woke && started.elapsed() < Duration::from_secs(1)); + let (state, entries) = target + .store + .read(&target.stream_ref(), 0, 10) + .await + .unwrap(); + assert!(state.closed); + assert_eq!((state.newest, entries.len()), (newest, 1)); + } +} + +/// The Records of one batch get consecutive offsets that end at the batch's last offset, so a +/// caller knows the offset of each one. +async fn a_batch_gets_consecutive_offsets_that_end_at_its_last_offset() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + append(&target, &client(), 1, &["before"]).await.unwrap(); + // Enough Records that `XADD *` on Redis 7.0 would cross a millisecond. + let texts: Vec = (0..1000).map(|i| i.to_string()).collect(); + let texts: Vec<&str> = texts.iter().map(String::as_str).collect(); + let appended = append(&target, &client(), 2, &texts).await.unwrap(); + let offsets: Vec = target.entries().await[1..] + .iter() + .map(|entry| entry.offset) + .collect(); + assert_eq!(offsets.len(), texts.len()); + assert_eq!( + appended.offsets, + Some((offsets[0], offsets[offsets.len() - 1])) + ); + assert!( + offsets.windows(2).all(|pair| pair[1] == pair[0] + 1), + "{offsets:?}" + ); + } +} + +/// A negative sequence is refused. +async fn a_negative_sequence_is_refused() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let error = append(&target, &client(), -1, &["x"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + assert!(target.texts().await.is_empty()); + } +} + +/// A batch with no sequence takes the next one of its Producer's epoch, and a newer epoch starts +/// at one. Such a batch has no dedupe, so its retry writes it again. A batch that sends the taken +/// number is a resend. +async fn a_batch_with_no_sequence_takes_the_next_one_of_its_epoch() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + let p = activity("r1", 1, 10); + append(&target, &p, 0, &["a"]).await.unwrap(); + let retry = append(&target, &p, 0, &["a"]).await.unwrap(); + let resend = append(&target, &p, 2, &["a"]).await.unwrap(); + assert_eq!(resend.offsets, retry.offsets); + let lower = append(&target, &p, 1, &["a"]).await.unwrap_err(); + assert_eq!(lower.reason, Reason::SequenceTooLow); + append(&target, &p, 7, &["b"]).await.unwrap(); + let next = append(&target, &p, 0, &["c"]).await.unwrap(); + let resend = append(&target, &p, 8, &["c"]).await.unwrap(); + assert_eq!(resend.offsets, next.offsets); + let newer = activity("r1", 2, 10); + let first = append(&target, &newer, 0, &["d"]).await.unwrap(); + let resend = append(&target, &newer, 1, &["d"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + assert_eq!(target.texts().await, ["a", "a", "b", "c", "d"]); + } +} + +/// Batches with no sequence that one Producer epoch sends at once each take their own number, so +/// the Store writes all of them. +async fn batches_with_no_sequence_at_once_each_take_their_own_number() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + let texts = ["t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8"]; + let appends = texts + .iter() + .map(|text| append(&target, &p, 0, std::slice::from_ref(text))); + for appended in join_all(appends).await { + appended.unwrap(); + } + let mut written = target.texts().await; + written.sort(); + assert_eq!(written, texts); + // The batches took 1 to 8, so 8 holds one of them. + let taken = append(&target, &p, 8, &["other"]).await.unwrap_err(); + assert_eq!(taken.reason, Reason::SeqDivergent); + } +} + +/// A large sequence keeps its dedupe entry, so a resend gets its first answer and a lower or +/// divergent batch is refused. A sequence past 2^53 - 1 is refused. +async fn a_large_sequence_dedupes_and_one_past_2_53_is_refused() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + let large = 100_000_000_000_000; + let first = append(&target, &p, large, &["x"]).await.unwrap(); + let resend = append(&target, &p, large, &["x"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + let divergent = append(&target, &p, large, &["other"]).await; + assert_eq!(divergent.unwrap_err().reason, Reason::SeqDivergent); + let lower = append(&target, &p, large - 1, &["lower"]).await; + assert_eq!(lower.unwrap_err().reason, Reason::SequenceTooLow); + assert_eq!(target.texts().await, ["x"]); + let max = (1 << 53) - 1; + append(&target, &p, max, &["max"]).await.unwrap(); + let past = append(&target, &p, max + 1, &["past"]).await; + assert_eq!(past.unwrap_err().reason, Reason::Invalid); + } +} + +/// An open Stream keeps a Producer's dedupe past the Retention. A resend of the last batch of a +/// Producer with no append for longer than the Retention gets the first answer, and the Stream +/// has one copy of the batch. +async fn a_silent_producer_resends_its_last_batch_once_past_the_retention() { + for target in stores_for(Needs::SHORT_TIMES).await { + target + .create_with(&Create { + retention: Some(Duration::from_millis(300)), + ..Default::default() + }) + .await + .unwrap(); + let p = client(); + let first = append(&target, &p, 1, &["old"]).await.unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + // A Store that dropped a silent Producer's state would do it on another Producer's append. + append(&target, &other_client(), 1, &["new"]).await.unwrap(); + let resend = append(&target, &p, 1, &["old"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + if target.gives.gives(Needs::STORE_ANSWERS) { + assert!(resend.resend); + } + assert_eq!(target.texts().await, ["old", "new"]); + } +} + +/// An open Stream keeps a Producer's fence past the Retention, so a lower epoch of a Producer with +/// no append for longer than the Retention is still fenced. +async fn a_silent_producer_stays_fenced_past_the_retention() { + for target in stores_for(Needs::SHORT_TIMES.and(Needs::UNCHECKED)).await { + target + .create_with(&Create { + retention: Some(Duration::from_millis(300)), + ..Default::default() + }) + .await + .unwrap(); + append(&target, &activity("r1", 2, 10), 1, &["new"]) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + append(&target, &other_client(), 1, &["other"]) + .await + .unwrap(); + let stale = append(&target, &activity("r1", 1, 10), 1, &["stale"]).await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + assert_eq!(target.texts().await, ["new", "other"]); + } +} + +/// A Producer with a lower epoch can't close the Stream. +async fn a_stale_producer_cant_close_the_stream() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + append(&target, &activity("r1", 2, 10), 1, &["a"]) + .await + .unwrap(); + assert_eq!( + target + .close(&activity("r1", 1, 10)) + .await + .unwrap_err() + .reason, + Reason::Fenced + ); + target.close(&activity("r1", 2, 10)).await.unwrap(); + } +} + +/// A Store refuses an append or a close of a Producer with no epoch, or with an epoch that has no +/// case, since the case gives the kind of the Producer. +async fn a_producer_with_no_epoch_case_is_refused() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let producer = Producer { + kind: None, + ..client() + }; + let error = append(&target, &producer, 1, &["a"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + let error = target.close(&producer).await.unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + assert!(target.texts().await.is_empty()); + } +} + +/// An append whose entry ids would not fit an offset is refused before it writes. +async fn an_append_whose_ids_would_not_fit_an_offset_is_refused_before_it_writes() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + // An entry a minute ahead of the clock, with nearly the last 20-bit sequence. + let ahead = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() + + 60_000; + target.add_raw_entry(&format!("{ahead}-1048570")).await; + let error = append(&target, &client(), 1, &["a"; 10]).await.unwrap_err(); + assert_eq!(error.reason, Reason::StorageUnavailable); + assert_eq!(target.log_len().await, 1); + } +} + +/// The entry ids of an append's offsets. +fn entry_ids(offsets: Option<(i64, i64)>) -> Option<(EntryId, EntryId)> { + let id = |offset: i64| { + let text = entry_id_of_offset(offset); + let (ms, seq) = text.split_once('-').unwrap(); + (ms.parse().unwrap(), seq.parse().unwrap()) + }; + offsets.map(|(first, last)| (id(first), id(last))) +} + +/// A Producer's `p|` field has the same 88 bytes on every Store. These are the fence, and the last +/// batch's sequence, entry ids and digest. +async fn a_producer_field_has_the_shared_layout() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let kind = ProducerKind::Activity as i32; + let field = |producer: &Producer, offsets, texts: &[&str]| { + ProducerField { + fence: epoch_key(producer.kind.as_ref()), + sequence: 1, + entries: entry_ids(offsets), + digest: digest(texts), + } + .encode() + }; + let a1 = activity("r1", 1, 10); + let stored = || async { target.meta(fields::producer(kind, &a1.id)).await.unwrap() }; + let written = append(&target, &a1, 1, &["x", "y"]).await.unwrap(); + assert_eq!(stored().await, field(&a1, written.offsets, &["x", "y"])); + // A newer attempt's empty batch sets its fence but writes no entry. + let a2 = activity("r1", 2, 20); + let empty = append(&target, &a2, 1, &[]).await.unwrap(); + assert_eq!(empty.offsets, None); + assert_eq!(stored().await, field(&a2, None, &[])); + } +} + +/// A Store refuses a write of a Producer whose `p|` field has another length, which a later format +/// may write. +async fn a_producer_field_of_another_format_is_refused() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let p = client(); + append(&target, &p, 1, &["a"]).await.unwrap(); + let field = fields::producer(ProducerKind::Client as i32, &p.id); + let value = target.meta(&field).await.unwrap(); + let longer = [value.clone(), vec![0]].concat(); + for other in [longer, value[..value.len() - 1].to_vec()] { + target.set_meta(&field, &other).await; + let error = append(&target, &p, 2, &["b"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::StorageCorrupt); + let error = target.close(&p).await.unwrap_err(); + assert_eq!(error.reason, Reason::StorageCorrupt); + } + assert_eq!(target.texts().await, ["a"]); + } +} + +/// The cases of the suite, in file order. +pub const CASES: &[super::Case] = &[ + case!(a_create_writes_the_key_once_and_keeps_its_retention), + case!(a_create_of_an_open_stream_fails_unless_it_uses_it_or_retries), + case!(closed_and_expired_come_before_already_exists), + case!(a_resend_gets_its_first_positions_and_a_lower_sequence_is_refused), + case!(a_resend_with_other_content_fails_as_divergent), + case!(a_producer_id_is_bytes_with_its_own_state), + case!(a_lower_epoch_is_fenced_for_each_producer_kind), + case!(a_close_stops_appends_but_not_a_resend), + case!(a_newer_format_refuses_every_write), + case!(an_expired_stream_refuses_writes_and_its_create), + case!(an_open_stream_with_no_last_write_ttl_never_expires_until_the_close), + case!(an_open_stream_keeps_its_records_past_its_retention), + case!(offsets_go_on_after_the_log_is_gone), + case!(a_blocked_read_wakes_on_an_append_and_ends_at_its_deadline), + case!(a_close_writes_no_entry_and_ends_a_wait), + case!(a_batch_gets_consecutive_offsets_that_end_at_its_last_offset), + case!(a_negative_sequence_is_refused), + case!(a_batch_with_no_sequence_takes_the_next_one_of_its_epoch), + case!(batches_with_no_sequence_at_once_each_take_their_own_number), + case!(a_large_sequence_dedupes_and_one_past_2_53_is_refused), + case!(a_silent_producer_resends_its_last_batch_once_past_the_retention), + case!(a_silent_producer_stays_fenced_past_the_retention), + case!(a_stale_producer_cant_close_the_stream), + case!(a_producer_with_no_epoch_case_is_refused), + case!(an_append_whose_ids_would_not_fit_an_offset_is_refused_before_it_writes), + case!(a_producer_field_has_the_shared_layout), + case!(a_producer_field_of_another_format_is_refused), +]; diff --git a/crates/streams/src/testing/support.rs b/crates/streams/src/testing/support.rs new file mode 100644 index 000000000..38bfbc1d3 --- /dev/null +++ b/crates/streams/src/testing/support.rs @@ -0,0 +1,274 @@ +//! The helpers that the suites share. They give a new Stream per case on each Store under test, +//! and the Producers of a Workflow run. + +#![allow(dead_code)] + +use super::{RawKeys, test_stores}; +use crate::{ + StreamResult, + digest::batch_digest, + proto::{ + AppendRecord, LocalActivityEpoch, LocalActivityProducer, Producer, StreamRecord, StreamRef, + WorkflowTaskEpoch, producer::Kind, + }, + store::{Appended, Batch, Closed, Create, Created, Entry, Store}, +}; +use prost::Message; +use prost_types::Timestamp; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +/// What a case needs beyond the Store's calls on its own Stream. A case runs on every Store that +/// has all that it needs. Each case's comment says its guarantee and its needs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Needs(u8); + +impl Needs { + /// Only the Store's calls on the Stream and reads of its Records. + pub const CALLS: Self = Self(0); + /// The Store's own results beyond the API. These are a create's stored Retention, `resend`, + /// `flagged`, and a close's offset and `already`. + pub const STORE_ANSWERS: Self = Self(1); + /// The keys under the calls. These are the meta fields, TTLs and raw log entries, and the + /// Store's log calls `read` and `wait_after`. + pub const RAW_KEYS: Self = Self(1 << 1); + /// Producers and runs as sent. The server checks them against its executions, so a + /// case with made-up runs can't run there. + pub const UNCHECKED: Self = Self(1 << 2); + /// A Retention of a few hundred ms. + pub const SHORT_TIMES: Self = Self(1 << 3); + /// The runtime built from the crate's own options, such as a Store name. + pub const OWN_CONFIG: Self = Self(1 << 4); + /// Every need. Redis and memory have all of them. + pub const ALL: Self = Self(u8::MAX); + + /// Both needs. + pub const fn and(self, other: Self) -> Self { + Self(self.0 | other.0) + } + + /// Whether this set gives every need of `needs`. + pub const fn gives(self, needs: Self) -> bool { + self.0 & needs.0 == needs.0 + } +} + +/// One Stream on one server, fresh for each case. +pub(crate) struct Target { + pub(crate) store: Arc, + pub(crate) raw: Arc, + pub(crate) stream: String, + /// What the Store gives. A case checks it before a step that needs more than it asked for. + pub(crate) gives: Needs, +} + +fn unique() -> String { + let n = NEXT.fetch_add(1, Ordering::Relaxed); + format!( + "{}-{n}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ) +} + +/// A new Stream on each Store under test that gives `needs`. +pub(crate) async fn stores_for(needs: Needs) -> Vec { + let mut targets = vec![]; + for test_store in test_stores() { + if !test_store.gives().gives(needs) { + continue; + } + let (store, raw) = test_store.connect().await; + let unique = unique(); + targets.push(Target { + store, + raw, + stream: format!("wf:{unique}"), + gives: test_store.gives(), + }); + } + targets +} + +/// The Producers of one Workflow run. +pub(crate) struct Run { + pub(crate) id: String, +} + +fn at(seconds: i64) -> Option { + Some(Timestamp { seconds, nanos: 0 }) +} + +impl Run { + pub(crate) fn new(id: &str) -> Self { + Self { id: id.to_string() } + } + + pub(crate) fn local_activity( + &self, + call: &str, + started: i64, + task_attempt: i32, + attempt: i32, + ) -> Producer { + let task = WorkflowTaskEpoch { + started_event_id: started, + attempt: task_attempt, + started_time: at(100), + }; + Producer { + id: format!("la:{}:{call}", self.id), + kind: Some(Kind::LocalActivity(LocalActivityProducer { + run_id: self.id.clone(), + epoch: Some(LocalActivityEpoch { + task: Some(task), + attempt, + }), + })), + } + } +} + +pub(crate) fn record(producer: &Producer, text: &str) -> Vec { + StreamRecord { + producer: Some(producer.clone()), + body: Some( + temporalio_common::protos::temporal::api::common::v1::Payload { + data: text.as_bytes().to_vec(), + ..Default::default() + }, + ), + ..Default::default() + } + .encode_to_vec() +} + +/// The bytes of a hex text, as the vectors give binary values. +pub(crate) fn hex(text: &str) -> Vec { + (0..text.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&text[at..at + 2], 16).unwrap()) + .collect() +} + +/// The digest of the batch that [record] makes of each text. +pub(crate) fn digest(texts: &[&str]) -> [u8; crate::digest::DIGEST_LEN] { + let records: Vec<_> = texts + .iter() + .map(|text| AppendRecord { + body: Some( + temporalio_common::protos::temporal::api::common::v1::Payload { + data: text.as_bytes().to_vec(), + ..Default::default() + }, + ), + }) + .collect(); + batch_digest(&records) +} + +/// The data texts of the entries, in log order. +pub(crate) fn records_of(entries: &[Entry]) -> Vec { + entries + .iter() + .filter_map(|entry| entry.record.as_ref()) + .map(|bytes| StreamRecord::decode(bytes.as_slice()).unwrap()) + .map(|record| String::from_utf8(record.body.unwrap().data).unwrap()) + .collect() +} + +impl Target { + pub(crate) fn stream_ref(&self) -> StreamRef { + StreamRef { + id: self.stream.clone(), + ..Default::default() + } + } + + pub(crate) async fn create(&self) { + self.create_with(&Create::default()).await.unwrap(); + } + + /// Creates the Stream, and gives the Store's answer. + pub(crate) async fn create_with(&self, create: &Create<'_>) -> StreamResult { + self.store.create(&self.stream_ref(), create).await + } + + /// Sets one meta field under the Store's calls. + pub(crate) async fn set_meta(&self, field: impl AsRef<[u8]>, value: impl AsRef<[u8]>) { + self.raw + .set_meta(&self.stream, field.as_ref(), value.as_ref()) + .await + } + + /// One meta field. + pub(crate) async fn meta(&self, field: impl AsRef<[u8]>) -> Option> { + self.raw.meta(&self.stream, field.as_ref()).await + } + + /// The ms left on the log and on the meta, as `PTTL` gives them. + pub(crate) async fn ttls(&self) -> (i64, i64) { + self.raw.ttls(&self.stream).await + } + + /// Adds a log entry with an explicit id, as `XADD `. + pub(crate) async fn add_raw_entry(&self, id: &str) { + self.raw.add_raw_entry(&self.stream, id).await + } + + /// The number of log entries. + pub(crate) async fn log_len(&self) -> usize { + self.raw.log_len(&self.stream).await + } + + /// Deletes the log, as an eviction would. + pub(crate) async fn drop_log(&self) { + self.raw.drop_log(&self.stream).await + } + + /// Every entry of the log. + pub(crate) async fn entries(&self) -> Vec { + let (_, entries) = self + .store + .read(&self.stream_ref(), 0, 10_000) + .await + .unwrap(); + entries + } + + /// The data texts in the log. + pub(crate) async fn texts(&self) -> Vec { + records_of(&self.entries().await) + } + + /// Appends a batch of `texts` as `producer` with `sequence`. + pub(crate) async fn batch( + &self, + producer: &Producer, + sequence: i64, + texts: &[&str], + ) -> StreamResult { + let records: Vec<_> = texts.iter().map(|text| record(producer, text)).collect(); + self.store + .append( + &self.stream_ref(), + &Batch { + producer, + sequence, + records: &records, + digest: digest(texts), + }, + ) + .await + } + + pub(crate) async fn close(&self, producer: &Producer) -> StreamResult { + self.store.close(&self.stream_ref(), producer).await + } +} From 42614ce4d9d4dfafe8c25afd34b46e6077efeebc Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Thu, 12 Nov 2026 21:56:37 -0700 Subject: [PATCH 3/5] Added the Stream runtime that answers the five operations by name. A bridge calls `StreamRuntime::call` with an operation name and the encoded request. The read path, the metrics and the shared service suite come with it, and the Store name is on each ref it builds. --- crates/streams/README.md | 40 ++ crates/streams/src/dispatch.rs | 172 +++++ crates/streams/src/lib.rs | 7 + crates/streams/src/metrics.rs | 51 ++ crates/streams/src/read.rs | 216 ++++++ crates/streams/src/record.rs | 118 ++++ crates/streams/src/runtime.rs | 268 ++++++++ crates/streams/src/testing/mod.rs | 30 +- crates/streams/src/testing/service.rs | 909 ++++++++++++++++++++++++++ crates/streams/src/testing/support.rs | 39 +- crates/streams/tests/capabilities.rs | 213 ++++++ crates/streams/tests/scans.rs | 206 ++++++ 12 files changed, 2266 insertions(+), 3 deletions(-) create mode 100644 crates/streams/README.md create mode 100644 crates/streams/src/dispatch.rs create mode 100644 crates/streams/src/metrics.rs create mode 100644 crates/streams/src/read.rs create mode 100644 crates/streams/src/record.rs create mode 100644 crates/streams/src/runtime.rs create mode 100644 crates/streams/src/testing/service.rs create mode 100644 crates/streams/tests/capabilities.rs create mode 100644 crates/streams/tests/scans.rs diff --git a/crates/streams/README.md b/crates/streams/README.md new file mode 100644 index 000000000..c4561e3d1 --- /dev/null +++ b/crates/streams/README.md @@ -0,0 +1,40 @@ +# temporalio-streams + +Experimental. The Stream runtime that the Temporal SDKs link into their bridges. It answers the five +operations of the Stream service on a Store. A bridge builds the runtime per client, as a plain +object, and calls `StreamRuntime::call` with an operation name and the encoded request. The call +gives back the encoded response. Core has no Stream code. + +## Protos and shared vectors + +The crate defines no protos. `temporalio-protos` generates them, and `temporalio_streams::proto` +re-exports them, so a value needs no conversion between the crates. + +- `temporal/api/stream/v1/message.proto`, `temporal/api/streamservice/v1/request_response.proto` + and `temporal/api/enums/v1/stream.proto` are the Stream protos of the API. They are in + `crates/protos/protos/api_upstream`. +- `testdata/*.json` are the shared vectors. They cover the Store keys of a Stream id, the meta + field of a Producer, offsets, epoch keys and the batch digest of a dedupe entry. + `testdata/gen_vectors.py` writes them from a second implementation, and the crate's tests read + each file. Each Store reads `fields.json` through the shared Store suite. `digests.json` is the + golden of the crate's batch digest. + +A Stream id and a Producer id are opaque bytes. The caller builds both. The runtime refuses an +empty id of either kind, and one over 4096 bytes. + +## The runtime + +`StreamRuntime` serves the Stream service on one Store, with one typed method for each operation. +`StreamRuntime::call` is the byte form of the same methods. It takes `CreateStream`, +`AppendRecords`, `CloseStream`, `ReadRecords` or `DescribeStream`, and any other name fails with +`Reason::Unsupported`. A Store crate builds the runtime with `StreamRuntime::new`, the name that +lang registered the Store under, and its Store. A `store::LazyStore` connects its inner Store on +the first call that needs it. So a bridge can build the runtime with no async runtime. +`MemoryStreamRuntime::new` builds it on the memory Store, from a `MemoryStreamOptions`. + +## Stores and the shared suites + +The memory Store is for tests. It keeps the key layout of the Redis Store and applies the same rules +in the same order. The `test-support` feature exports the shared suites in `testing`. A Store crate +implements `testing::TestStore` for its Store and runs each suite with `testing::run`, so that +every Store keeps one set of rules. This crate runs them on the memory Store in its unit tests. diff --git a/crates/streams/src/dispatch.rs b/crates/streams/src/dispatch.rs new file mode 100644 index 000000000..6acec56f5 --- /dev/null +++ b/crates/streams/src/dispatch.rs @@ -0,0 +1,172 @@ +//! The byte form of the runtime's calls, for bridges. A bridge passes the encoded request and gets +//! the encoded response, so it decodes no proto. + +use crate::{Reason, StreamError, StreamResult, StreamRuntime}; +use prost::Message; + +fn decode(input: &[u8]) -> StreamResult { + M::decode(input).map_err(|error| StreamError::invalid(format!("The request is bad: {error}"))) +} + +impl StreamRuntime { + /// Runs one of the five Stream service operations on an encoded request, by its name, and + /// returns the encoded response. The names are `CreateStream`, `AppendRecords`, `CloseStream`, + /// `ReadRecords` and `DescribeStream`. The bridge raises an error with [StreamError::code] and + /// [StreamError::details]. + pub async fn call(&self, operation: &str, request: &[u8]) -> StreamResult> { + let response = match operation { + "CreateStream" => self.create_stream(decode(request)?).await?.encode_to_vec(), + "AppendRecords" => self.append_records(decode(request)?).await?.encode_to_vec(), + "CloseStream" => self.close_stream(decode(request)?).await?.encode_to_vec(), + "ReadRecords" => self.read_records(decode(request)?).await?.encode_to_vec(), + "DescribeStream" => self + .describe_stream(decode(request)?) + .await? + .encode_to_vec(), + _ => { + return Err(StreamError::new( + Reason::Unsupported, + format!("Unknown operation {operation}"), + )); + } + }; + Ok(response) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + MemoryStreamOptions, MemoryStreamRuntime, + proto::{ + AppendRecord, AppendRecordsRequest, AppendRecordsResponse, ClientProducer, + CloseStreamRequest, CloseStreamResponse, CreateStreamRequest, CreateStreamResponse, + DescribeStreamRequest, DescribeStreamResponse, Producer, ReadRecordsRequest, + ReadRecordsResponse, StreamRef, StreamStatus, producer::Kind, + }, + }; + use std::sync::Arc; + use temporalio_common::protos::temporal::api::common::v1::Payload; + use tonic::Code; + + const OPERATIONS: [&str; 5] = [ + "CreateStream", + "AppendRecords", + "CloseStream", + "ReadRecords", + "DescribeStream", + ]; + + fn runtime() -> Arc { + MemoryStreamRuntime::new("b", MemoryStreamOptions::default(), None) + } + + fn stream_ref() -> Option { + Some(StreamRef { + id: "s".to_string(), + ..Default::default() + }) + } + + fn client() -> Option { + Some(Producer { + id: "client:me:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + }) + } + + fn records() -> Vec { + vec![AppendRecord { + body: Some(Payload::default()), + }] + } + + async fn call( + runtime: &StreamRuntime, + operation: &str, + request: Req, + ) -> Resp { + let bytes = runtime + .call(operation, &request.encode_to_vec()) + .await + .unwrap(); + Resp::decode(bytes.as_slice()).unwrap() + } + + async fn create(runtime: &StreamRuntime) -> CreateStreamResponse { + let request = CreateStreamRequest { + stream_ref: stream_ref(), + ..Default::default() + }; + call(runtime, "CreateStream", request).await + } + + // A typo in an operation name would make it unknown, which is unsupported, so the loop also + // checks the names that a bridge calls. + #[tokio::test] + async fn bad_bytes_are_invalid_and_an_unknown_operation_is_unsupported() { + let runtime = runtime(); + for operation in OPERATIONS { + let error = runtime.call(operation, &[0xff]).await.unwrap_err(); + assert_eq!(error.reason, Reason::Invalid, "{operation}"); + } + let error = runtime.call("Nope", &[]).await.unwrap_err(); + assert_eq!( + (error.reason, error.code(), error.message.as_str()), + ( + Reason::Unsupported, + Code::Unimplemented, + "Unknown operation Nope" + ) + ); + } + + #[tokio::test] + async fn each_stream_operation_round_trips_its_bytes() { + let runtime = runtime(); + assert_eq!(create(&runtime).await.r#ref.unwrap().store, "b"); + let appended: AppendRecordsResponse = call( + &runtime, + "AppendRecords", + AppendRecordsRequest { + r#ref: stream_ref(), + producer: client(), + sequence: 1, + records: records(), + ..Default::default() + }, + ) + .await; + let read: ReadRecordsResponse = call( + &runtime, + "ReadRecords", + ReadRecordsRequest { + r#ref: stream_ref(), + ..Default::default() + }, + ) + .await; + assert_eq!(read.last_offset, appended.last_offset); + let _: CloseStreamResponse = call( + &runtime, + "CloseStream", + CloseStreamRequest { + r#ref: stream_ref(), + producer: client(), + ..Default::default() + }, + ) + .await; + let described: DescribeStreamResponse = call( + &runtime, + "DescribeStream", + DescribeStreamRequest { + r#ref: stream_ref(), + ..Default::default() + }, + ) + .await; + assert_eq!(described.status, StreamStatus::Closed as i32); + } +} diff --git a/crates/streams/src/lib.rs b/crates/streams/src/lib.rs index 747256328..fffcb54c2 100644 --- a/crates/streams/src/lib.rs +++ b/crates/streams/src/lib.rs @@ -6,16 +6,23 @@ //! builds the runtime and calls it with an operation name and the encoded request. pub mod digest; +mod dispatch; pub mod epoch; mod error; pub mod ids; pub mod memory; +mod metrics; pub mod offset; +mod read; +mod record; +mod runtime; pub mod store; #[cfg(any(test, feature = "test-support"))] pub mod testing; pub use error::{Reason, StreamError, StreamResult}; +pub use read::MAX_WAIT; +pub use runtime::{MemoryStreamOptions, MemoryStreamRuntime, StreamRuntime}; /// The protos that lang and the crate exchange. `temporalio-protos` generates them, so a value /// needs no conversion between the crates. diff --git a/crates/streams/src/metrics.rs b/crates/streams/src/metrics.rs new file mode 100644 index 000000000..5fa5fb14d --- /dev/null +++ b/crates/streams/src/metrics.rs @@ -0,0 +1,51 @@ +//! The crate's metrics. They use the meter that lang gives, or a no-op meter. + +use temporalio_common::telemetry::metrics::{ + Counter, MetricAttributes, MetricParameters, TemporalMeter, UpDownCounter, +}; + +#[derive(Clone)] +pub(crate) struct Metrics { + attributes: MetricAttributes, + fenced: Counter, + blocking_reads_waiting: UpDownCounter, +} + +fn parameters( + name: &'static str, + description: &'static str, + unit: &'static str, +) -> MetricParameters { + MetricParameters { + name: name.into(), + description: description.into(), + unit: unit.into(), + } +} + +impl Metrics { + pub(crate) fn new(meter: &TemporalMeter) -> Self { + Self { + attributes: meter.get_default_attributes().clone(), + fenced: meter.counter(parameters( + "stream_fenced", + "Batches that the Store refused for a lower epoch", + "", + )), + blocking_reads_waiting: meter.up_down_counter(parameters( + "stream_blocking_reads_waiting", + "Reads that wait on the Store", + "", + )), + } + } + + pub(crate) fn fenced(&self) { + self.fenced.add(1, &self.attributes); + } + + pub(crate) fn read_waits(&self, waiting: bool) { + self.blocking_reads_waiting + .add(if waiting { 1 } else { -1 }, &self.attributes); + } +} diff --git a/crates/streams/src/read.rs b/crates/streams/src/read.rs new file mode 100644 index 000000000..31f9b47db --- /dev/null +++ b/crates/streams/src/read.rs @@ -0,0 +1,216 @@ +//! The read of the runtime, a long poll that pages through the log. + +use crate::{ + Reason, StreamError, StreamResult, StreamRuntime, + offset::{offset_from_wire, offset_to_wire}, + proto::{ReadRecordsRequest, ReadRecordsResponse, StreamRecord, StreamRef}, + runtime::{duration, stream_id}, + store::{FORMAT_VERSION, StreamState}, +}; +use prost::Message; +use std::time::Duration; +use tokio::time::Instant; + +/// The longest wait of one read for new Records. +pub const MAX_WAIT: Duration = Duration::from_secs(60); +/// The maximum number of Records in one read. +const MAX_RECORDS: usize = 1000; + +impl StreamRuntime { + /// A long poll that returns the Records it found, or nothing after `wait`. On a closed Stream + /// with no Record left, it returns `done` at once. + pub async fn read_records( + &self, + request: ReadRecordsRequest, + ) -> StreamResult { + stream_id(request.r#ref.as_ref())?; + let stream = request.r#ref.clone().unwrap_or_default(); + // The wire from_offset is exclusive, so a read starts after it. The empty value is a read + // from the start. + let from_offset = offset_from_wire(&request.from_offset)?; + if from_offset < 0 { + return Err(StreamError::invalid("The offset is below zero")); + } + let max_bytes = max_bytes(request.max_bytes)?; + let max = match request.max_records { + n if n <= 0 => MAX_RECORDS, + n => (n as usize).min(MAX_RECORDS), + }; + let started = Instant::now(); + let wait = duration(request.wait).unwrap_or_default().min(MAX_WAIT); + let deadline = started + wait; + + // A creator hands out the ref only after its create, so a read never waits for one. + let (mut state, _) = self.store.read(&stream, 0, 0).await?; + if !state.exists { + return Err(StreamError::new( + Reason::NotFound, + "Nobody created the Stream", + )); + } + if state.version > FORMAT_VERSION { + return Err(StreamError::new( + Reason::FormatTooNew, + format!("format {} is newer than {FORMAT_VERSION}", state.version), + )); + } + // Before the start is one past the offset, since the largest offset has no next one. + if from_offset > state.newest { + return Err(StreamError::new( + Reason::OffsetTooHigh, + "The offset is past the newest Record. A failover may have lost the newest \ + Records. Resume from the newest offset, or read again once newer Records pass it", + )); + } + // An exclusive from_offset reads after that Record, so the inclusive start is one past it. + let from = match from_offset { + 0 => 0, + offset => offset + 1, + }; + loop { + check_expiry(&state, from)?; + if let Some(page) = self.page(&stream, &state, from, (max, max_bytes)).await? { + return Ok(page); + } + if deadline <= Instant::now() { + // An empty read resumes from where it started, which is one before the inclusive + // start. + return Ok(ReadRecordsResponse { + last_offset: offset_to_wire(from - 1), + ..Default::default() + }); + } + let after = (from - 1).max(state.newest); + self.metrics.read_waits(true); + let woke = self.store.wait_after(&stream, after, deadline).await; + self.metrics.read_waits(false); + woke?; + state = self.store.read(&stream, 0, 0).await?.0; + } + } + + /// The Records from `from`, or `None` when there is none yet and the Stream is open. A page + /// of a closed Stream that gives its last Record is done, also with no Record. The pair + /// `(max, max_bytes)` caps the Records and their bytes. + async fn page( + &self, + stream: &StreamRef, + state: &StreamState, + from: i64, + (max, max_bytes): (usize, usize), + ) -> StreamResult> { + // A read from the start skips the Records that the log lost. A read from an offset fails on + // them, since the Reader asked for those Records. + let retained = state.retained_from(); + let mut start = if from > 0 { from } else { retained }; + let mut records = vec![]; + let mut full = false; + let mut bytes = 0; + // The bytes and the entries that the page read from the Store. + let mut pulled = (0, 0); + // Whether the Stream was closed, and its newest offset, at the newest Store read. + let mut end = (state.closed, state.newest); + while records.len() < max && !full { + let room = (max_bytes != usize::MAX).then(|| max_bytes.saturating_sub(bytes)); + let chunk = chunk_len(max - records.len(), room, pulled); + let (chunk_state, entries) = self.store.read(stream, start, chunk).await?; + // Records that the log lost after the state read leave a gap before these entries. + check_expiry(&chunk_state, if from > 0 { start } else { 0 })?; + end = (chunk_state.closed, chunk_state.newest); + if entries.is_empty() { + // Only an expired log has no entry at an offset up to `newest`. + if from > 0 && start == from && from <= chunk_state.newest { + return Err(expired()); + } + break; + } + for entry in entries { + if entry.offset < retained { + if from > 0 { + return Err(expired()); + } + start = entry.offset + 1; + continue; + } + let Some(encoded) = entry.record else { + start = entry.offset + 1; + continue; + }; + let record = decode_record(&encoded, entry.offset)?; + // An entry over the caps is still returned alone, so that the read always makes + // progress. The next read starts at the entry that didn't fit. + let size = record.encoded_len(); + pulled = (pulled.0 + size, pulled.1 + 1); + let over = bytes + size > max_bytes || records.len() + 1 > max; + if !records.is_empty() && over { + full = true; + break; + } + bytes += size; + records.push(record); + start = entry.offset + 1; + if records.len() >= max { + break; + } + } + } + // A closed Stream takes no more appends, so no Record is left past its newest offset. + let done = end.0 && start > end.1; + if records.is_empty() && !done { + return Ok(None); + } + // The resume is after the last Record returned, which is one before the inclusive start. + let page = ReadRecordsResponse { + records, + last_offset: offset_to_wire(start - 1), + done, + }; + Ok(Some(page)) + } +} + +/// The number of entries of the next Store read, at most `left`. With a byte cap, the first read +/// takes one entry, and each later one takes what fits the `room` left at the average size so far. +fn chunk_len(left: usize, room: Option, (bytes, entries): (usize, usize)) -> usize { + let Some(room) = room else { + return left; + }; + if entries == 0 { + return 1; + } + let average = (bytes / entries).max(1); + (room / average + 1).clamp(1, left.max(1)) +} + +fn decode_record(bytes: &[u8], offset: i64) -> StreamResult { + let mut record = StreamRecord::decode(bytes).map_err(|error| { + StreamError::storage_corrupt(format!("A stored Record is bad: {error}")) + })?; + record.offset = offset_to_wire(offset); + Ok(record) +} + +/// Fails a read whose Records the log lost or the expiry removed, so that the Reader always sees +/// the gap. A read from the start of the Stream has no gap. +/// +/// A Tombstone fails every read. An open Stream that expired gets no more Records, so the Reader +/// would wait forever. A closed one lost its log. +fn check_expiry(state: &StreamState, from: i64) -> StreamResult<()> { + if state.expired() || (from > 0 && state.trimmed >= from) { + return Err(expired()); + } + Ok(()) +} + +/// The byte cap of a page, from a request's `max_bytes`. Zero is no cap. +fn max_bytes(value: i64) -> StreamResult { + match value { + 0 => Ok(usize::MAX), + n if n < 0 => Err(StreamError::invalid("max_bytes must be 0 or more")), + n => Ok(usize::try_from(n).unwrap_or(usize::MAX)), + } +} + +fn expired() -> StreamError { + StreamError::new(Reason::Expired, "The Records at the offset expired") +} diff --git a/crates/streams/src/record.rs b/crates/streams/src/record.rs new file mode 100644 index 000000000..ca0607a74 --- /dev/null +++ b/crates/streams/src/record.rs @@ -0,0 +1,118 @@ +//! The stored Records of a batch. + +use crate::{ + Reason, StreamError, StreamResult, + proto::{AppendRecord, Producer, StreamRecord}, + store::{MAX_BATCH_BYTES, MAX_BATCH_RECORDS, MAX_RECORD_BYTES}, +}; +use prost::Message; + +/// One stored data Record. It has no offset, since the Store gives the entry its offset. +pub(crate) fn data_record(producer: &Producer, record: AppendRecord) -> Vec { + StreamRecord { + producer: Some(producer.clone()), + body: record.body, + ..Default::default() + } + .encode_to_vec() +} + +/// Refuses a batch with a Record that has no body, before the Store. +pub(crate) fn check_records(records: &[AppendRecord]) -> StreamResult<()> { + match records.iter().position(|record| record.body.is_none()) { + Some(at) => Err(StreamError::invalid(format!( + "Record {at} of the batch has no body" + ))), + None => Ok(()), + } +} + +/// The stored Records of one batch from outside Workflow code. A batch over the Store's limits +/// fails, since the crate doesn't split a batch it didn't build. +pub(crate) fn data_records( + producer: &Producer, + records: Vec, +) -> StreamResult>> { + if records.len() > MAX_BATCH_RECORDS { + return Err(StreamError::new( + Reason::TooLarge, + format!( + "The batch has {} Records, more than {MAX_BATCH_RECORDS}", + records.len() + ), + )); + } + let records: Vec<_> = records + .into_iter() + .map(|record| data_record(producer, record)) + .collect(); + if let Some(big) = records + .iter() + .find(|record| record.len() > MAX_RECORD_BYTES) + { + return Err(StreamError::new( + Reason::TooLarge, + format!( + "A Record has {} bytes, more than {MAX_RECORD_BYTES}", + big.len() + ), + )); + } + let total: usize = records.iter().map(Vec::len).sum(); + if total > MAX_BATCH_BYTES { + return Err(StreamError::new( + Reason::TooLarge, + format!("The batch has {total} bytes, more than {MAX_BATCH_BYTES}"), + )); + } + Ok(records) +} + +#[cfg(test)] +mod tests { + use super::*; + use temporalio_common::protos::temporal::api::common::v1::Payload; + + #[test] + fn a_record_with_no_body_is_invalid() { + let body = AppendRecord { + body: Some(Payload::default()), + }; + check_records(std::slice::from_ref(&body)).unwrap(); + let error = check_records(&[body, AppendRecord::default()]).unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + assert!(error.message.contains("Record 1"), "{}", error.message); + } + + #[test] + fn a_batch_over_the_limits_fails() { + let producer = Producer::default(); + let record = |bytes: usize| AppendRecord { + body: Some(Payload { + data: vec![0; bytes], + ..Default::default() + }), + }; + let error = data_records(&producer, vec![record(MAX_RECORD_BYTES)]).unwrap_err(); + assert_eq!(error.reason, Reason::TooLarge); + let many = (0..4) + .map(|_| record(MAX_RECORD_BYTES / 2 + 1)) + .collect::>(); + let many = [many.clone(), many].concat(); + assert_eq!( + data_records(&producer, many).unwrap_err().reason, + Reason::TooLarge + ); + assert_eq!(data_records(&producer, vec![record(10)]).unwrap().len(), 1); + let small = vec![record(1); MAX_BATCH_RECORDS + 1]; + assert_eq!( + data_records(&producer, small).unwrap_err().reason, + Reason::TooLarge + ); + let small = vec![record(1); MAX_BATCH_RECORDS]; + assert_eq!( + data_records(&producer, small).unwrap().len(), + MAX_BATCH_RECORDS + ); + } +} diff --git a/crates/streams/src/runtime.rs b/crates/streams/src/runtime.rs new file mode 100644 index 000000000..dc590d911 --- /dev/null +++ b/crates/streams/src/runtime.rs @@ -0,0 +1,268 @@ +//! The Stream runtime, a plain object that lang's bridge builds per client. + +use crate::{ + Reason, StreamError, StreamResult, + digest::batch_digest, + epoch::ProducerKind, + ids::{check_producer, check_required, check_stream_id}, + memory::MemoryStore, + metrics::Metrics, + offset::offset_to_wire, + proto::{ + AppendRecordsRequest, AppendRecordsResponse, CloseStreamRequest, CloseStreamResponse, + CreateStreamRequest, CreateStreamResponse, DescribeStreamRequest, DescribeStreamResponse, + Producer, StreamIdConflictPolicy, StreamRef, StreamStatus, + }, + record::{check_records, data_records}, + store::{ + Batch, Capabilities, ConflictPolicy, Create, CreateOutcome, DEFAULT_RETENTION, + MAX_SEQUENCE, Store, + }, +}; +use std::{sync::Arc, time::Duration}; +use temporalio_common::telemetry::metrics::TemporalMeter; + +/// The Stream runtime on one Store. It serves the five Stream service operations. A Store has no +/// namespaces, so the runtime ignores a request's `namespace`. The SDK sets it, and only the +/// server reads it. +/// +/// A bridge builds one for each client with a Stream Store, and that client's Workers share it. +/// A Store crate, such as `temporalio-streams-redis`, builds it with its own Store. +pub struct StreamRuntime { + /// The name of the Store, which the refs that the runtime builds carry. + store_name: String, + pub(crate) store: Arc, + pub(crate) metrics: Metrics, +} + +impl std::fmt::Debug for StreamRuntime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StreamRuntime") + .field("store_name", &self.store_name) + .finish_non_exhaustive() + } +} + +pub(crate) fn duration(value: Option) -> Option { + let value = value?; + let duration = Duration::new(value.seconds.max(0) as u64, value.nanos.max(0) as u32); + (!duration.is_zero()).then_some(duration) +} + +/// The conflict policy of a create. Unspecified is FAIL, as for a Workflow start. +fn conflict_policy(value: i32) -> StreamResult { + match StreamIdConflictPolicy::try_from(value) { + Ok(StreamIdConflictPolicy::Unspecified | StreamIdConflictPolicy::Fail) => { + Ok(ConflictPolicy::Fail) + } + Ok(StreamIdConflictPolicy::UseExisting) => Ok(ConflictPolicy::UseExisting), + Err(_) => Err(StreamError::invalid(format!( + "The conflict policy {value} is unknown" + ))), + } +} + +pub(crate) fn stream_id(stream_ref: Option<&StreamRef>) -> StreamResult<&str> { + let id = stream_ref + .map(|stream_ref| stream_ref.id.as_str()) + .unwrap_or_default(); + check_stream_id(id)?; + Ok(id) +} + +/// Checks the Producer of a close. Any kind may close. +fn close_producer(producer: Option<&Producer>) -> StreamResult<&Producer> { + let producer = producer.ok_or_else(|| StreamError::invalid("The request has no Producer"))?; + check_producer(producer)?; + check_required(producer)?; + Ok(producer) +} + +impl StreamRuntime { + /// Builds a runtime on `store`. `store_name` is the name that lang registered the Store under, + /// which the refs that the runtime builds carry. The runtime records metrics on `meter`, lang's + /// runtime meter, if set. A Store that connects later, such as a [crate::store::LazyStore], + /// lets a bridge build the runtime with no async runtime. + pub fn new( + store_name: impl Into, + store: Arc, + meter: Option, + ) -> Arc { + let meter = meter.unwrap_or_else(TemporalMeter::no_op); + Arc::new(Self { + store_name: store_name.into(), + store, + metrics: Metrics::new(&meter), + }) + } + + /// Creates a Stream, or gets it with USE_EXISTING. + pub async fn create_stream( + &self, + request: CreateStreamRequest, + ) -> StreamResult { + stream_id(request.stream_ref.as_ref())?; + let stream = request.stream_ref.clone().unwrap_or_default(); + let conflict = conflict_policy(request.id_conflict_policy)?; + // The Store config holds the Retention, so a create takes the Store default. + let create = Create { + retention: None, + conflict, + request_id: &request.request_id, + }; + let created = self.store.create(&stream, &create).await?; + Ok(CreateStreamResponse { + r#ref: Some(StreamRef { + id: stream.id, + store: self.store_name.clone(), + }), + created: created.outcome == CreateOutcome::Created, + }) + } + + /// Appends one batch as one Producer. A Workflow appends as the Producer of its Local Activity, + /// so a Workflow Producer is refused. + pub async fn append_records( + &self, + request: AppendRecordsRequest, + ) -> StreamResult { + stream_id(request.r#ref.as_ref())?; + let stream = request.r#ref.clone().unwrap_or_default(); + let producer = request + .producer + .as_ref() + .ok_or_else(|| StreamError::invalid("The request has no Producer"))?; + check_producer(producer)?; + if ProducerKind::try_from(producer)? == ProducerKind::Workflow { + return Err(StreamError::invalid( + "A Workflow writes with the Producer of its Local Activity", + )); + } + check_required(producer)?; + if !(0..=MAX_SEQUENCE).contains(&request.sequence) { + return Err(StreamError::invalid( + "The sequence must be from 1 to 2^53 - 1, or zero for the next one", + )); + } + check_records(&request.records)?; + // Without the fence, a stale batch would reach Readers. + let capabilities = self.store.capabilities(); + capabilities.require(Capabilities::PRODUCER_FENCE)?; + if request.sequence == 0 { + // Only the Store can number the batch in the same step as its dedupe check. + capabilities.require(Capabilities::NEXT_SEQUENCE)?; + } + let digest = batch_digest(&request.records); + let records = data_records(producer, request.records)?; + let batch = Batch { + producer, + sequence: request.sequence, + records: &records, + digest, + }; + let appended = self.store.append(&stream, &batch).await; + if let Err(error) = &appended + && error.reason == Reason::Fenced + { + self.metrics.fenced(); + } + // An empty batch has no offset, which is the empty wire value. + let (first_offset, last_offset) = appended?.offsets.unwrap_or_default(); + Ok(AppendRecordsResponse { + first_offset: offset_to_wire(first_offset), + last_offset: offset_to_wire(last_offset), + }) + } + + /// Closes a Stream. Its readers drain what is left, and later appends fail. + pub async fn close_stream( + &self, + request: CloseStreamRequest, + ) -> StreamResult { + stream_id(request.r#ref.as_ref())?; + let stream = request.r#ref.clone().unwrap_or_default(); + let producer = close_producer(request.producer.as_ref())?; + self.store.capabilities().require(Capabilities::CLOSE)?; + self.store.close(&stream, producer).await?; + Ok(CloseStreamResponse {}) + } + + /// Gets the status and the newest offset of a Stream. + pub async fn describe_stream( + &self, + request: DescribeStreamRequest, + ) -> StreamResult { + stream_id(request.r#ref.as_ref())?; + let stream = request.r#ref.clone().unwrap_or_default(); + let (state, _) = self.store.read(&stream, 0, 0).await?; + if !state.exists { + return Err(StreamError::new( + Reason::NotFound, + "Nobody created the Stream", + )); + } + // A closed Stream that expired has lost its log, and every read fails. + let status = if state.expired() { + StreamStatus::Expired + } else if state.closed { + StreamStatus::Closed + } else { + StreamStatus::Open + }; + Ok(DescribeStreamResponse { + status: status as i32, + last_offset: offset_to_wire(state.newest), + }) + } +} + +/// The settings of the memory Store runtime. A field left `None` takes its default. +#[derive(Clone, Debug, Default)] +pub struct MemoryStreamOptions { + /// The Retention of each Stream. `None` is [DEFAULT_RETENTION]. + pub retention: Option, + /// How long an open Stream stays after its last write. `None` keeps an open Stream until its + /// close. + pub last_write_ttl: Option, +} + +/// The Stream runtime on the memory Store, for tests. It serves the same calls with the same +/// rules as every other Store, and it keeps its Streams in this process only. +#[derive(Debug)] +pub struct MemoryStreamRuntime; + +impl MemoryStreamRuntime { + /// Builds a runtime on a new memory Store. A zero duration in `options` is the same as `None`. + /// The type only names this constructor, so it returns the runtime and not `Self`. + #[allow(clippy::new_ret_no_self)] + pub fn new( + store_name: impl Into, + options: MemoryStreamOptions, + meter: Option, + ) -> Arc { + let positive = |value: Option| value.filter(|value| !value.is_zero()); + let store = Arc::new(MemoryStore::new( + positive(options.retention).unwrap_or(DEFAULT_RETENTION), + positive(options.last_write_ttl), + )); + StreamRuntime::new(store_name, store, meter) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_zero_or_negative_duration_is_unset_and_a_huge_one_stays() { + let proto = |seconds, nanos| Some(prost_types::Duration { seconds, nanos }); + assert_eq!(duration(None), None); + assert_eq!(duration(proto(0, 0)), None); + assert_eq!(duration(proto(-5, 0)), None); + assert_eq!(duration(proto(0, -5)), None); + assert_eq!( + duration(proto(i64::MAX, 999_999_999)), + Some(Duration::new(i64::MAX as u64, 999_999_999)) + ); + } +} diff --git a/crates/streams/src/testing/mod.rs b/crates/streams/src/testing/mod.rs index b60561ad3..0879aeffe 100644 --- a/crates/streams/src/testing/mod.rs +++ b/crates/streams/src/testing/mod.rs @@ -4,16 +4,18 @@ mod support; pub mod fields; +pub mod service; pub mod store; pub use support::Needs; use crate::{ + MemoryStreamOptions, MemoryStreamRuntime, StreamRuntime, memory::MemoryStore, store::{DEFAULT_RETENTION, Store}, }; use futures_util::{FutureExt, future::join_all}; -use std::{future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc}; +use std::{future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::Duration}; /// The state under a Store's calls, which some cases read or change. #[async_trait::async_trait] @@ -32,6 +34,17 @@ pub trait RawKeys: Send + Sync { async fn drop_log(&self, stream: &str); } +/// The settings of a runtime under test. A field left `None` keeps the Store's own value. +#[derive(Clone, Debug, Default)] +pub struct RuntimeConfig { + /// The Store name that the runtime sets on each ref it builds. + pub store_name: String, + /// The Retention of each Stream that the runtime creates. + pub retention: Option, + /// How long an open Stream stays after its last write. + pub last_write_ttl: Option, +} + /// A Store that the suites run on, with the server under it. #[async_trait::async_trait] pub trait TestStore: Send + Sync { @@ -39,6 +52,8 @@ pub trait TestStore: Send + Sync { fn gives(&self) -> Needs; /// A Store for one case, and the state under it. async fn connect(&self) -> (Arc, Arc); + /// A runtime on the Store with `config`. + fn runtime(&self, config: &RuntimeConfig) -> Arc; } /// The memory Store, with a new Store for each case. @@ -55,6 +70,14 @@ impl TestStore for MemoryTestStore { let store = Arc::new(MemoryStore::new(DEFAULT_RETENTION, None)); (store.clone(), store) } + + fn runtime(&self, config: &RuntimeConfig) -> Arc { + let options = MemoryStreamOptions { + retention: config.retention, + last_write_ttl: config.last_write_ttl, + }; + MemoryStreamRuntime::new(config.store_name.clone(), options, None) + } } #[async_trait::async_trait] @@ -139,4 +162,9 @@ mod tests { async fn the_store_suite_passes_on_the_memory_store() { run(vec![Arc::new(MemoryTestStore)], store::CASES).await; } + + #[tokio::test] + async fn the_service_suite_passes_on_the_memory_store() { + run(vec![Arc::new(MemoryTestStore)], service::CASES).await; + } } diff --git a/crates/streams/src/testing/service.rs b/crates/streams/src/testing/service.rs new file mode 100644 index 000000000..393cc9bce --- /dev/null +++ b/crates/streams/src/testing/service.rs @@ -0,0 +1,909 @@ +//! The Stream service operations through the runtime's typed calls. +//! +//! Every Store runs the suite through its own runtime. + +use super::{RuntimeConfig, case, support::Needs}; + +use crate::{ + StreamError, StreamRuntime, + ids::MAX_PRODUCER_ID_BYTES, + proto::{ + ActivityEpoch, ActivityProducer, AppendRecord, AppendRecordsRequest, AppendRecordsResponse, + ClientProducer, CloseStreamRequest, CloseStreamResponse, CreateStreamRequest, + CreateStreamResponse, DescribeStreamRequest, DescribeStreamResponse, LocalActivityEpoch, + LocalActivityProducer, Producer, ReadRecordsRequest, ReadRecordsResponse, + StreamErrorDetails, StreamIdConflictPolicy, StreamRef, StreamStatus, WorkflowProducer, + WorkflowTaskEpoch, producer::Kind, + }, +}; +use prost::Message; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use temporalio_common::protos::temporal::api::common::v1::Payload; +use tonic::Code; + +struct Case { + ext: Arc, + namespace: String, + stream: String, +} + +static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +fn unique() -> String { + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!("{now}-{n}") +} + +/// The Retention of the cases that let a closed Stream expire. +const SHORT_RETENTION: Duration = Duration::from_millis(300); + +/// A fresh Stream on each Store that gives `needs`. +async fn cases_for(needs: Needs) -> Vec { + cases_with(needs, &RuntimeConfig::default()).await +} + +/// A fresh Stream on each Store that gives short times, on a runtime whose Retention is +/// [SHORT_RETENTION]. +async fn short_cases() -> Vec { + let config = RuntimeConfig { + retention: Some(SHORT_RETENTION), + ..Default::default() + }; + cases_with(Needs::SHORT_TIMES, &config).await +} + +/// A fresh Stream on each Store that gives `needs`, on a runtime with `config`. +async fn cases_with(needs: Needs, config: &RuntimeConfig) -> Vec { + super::support::runtimes(needs, config) + .await + .into_iter() + .map(|runtime| { + let unique = unique(); + Case { + ext: runtime.ext, + namespace: runtime.namespace.unwrap_or(format!("ns-{unique}")), + stream: format!("s-{unique}"), + } + }) + .collect() +} + +fn reason(error: &StreamError) -> String { + let details = StreamErrorDetails::decode(error.details().as_slice()).unwrap(); + assert_eq!(details.reason(), error.reason.proto()); + error.reason.token().to_string() +} + +/// The internal offset of a wire value, for the suite's assertions. +fn off(wire: &[u8]) -> i64 { + crate::offset::offset_from_wire(wire).unwrap() +} + +/// The wire value of an internal offset. +fn wire(offset: i64) -> Vec { + crate::offset::offset_to_wire(offset) +} + +fn client() -> Producer { + Producer { + id: "client:me:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } +} + +/// The Producer of run `run` in its first Workflow Task, as its close Activity sends it. +fn workflow(run: &str) -> Producer { + Producer { + id: format!("wf:{run}"), + kind: Some(Kind::Workflow(WorkflowProducer { + run_id: run.to_string(), + epoch: Some(WorkflowTaskEpoch { + started_event_id: 3, + attempt: 1, + started_time: None, + }), + })), + } +} + +/// The fields of a create that uses an existing Stream. +fn use_existing() -> CreateStreamRequest { + CreateStreamRequest { + id_conflict_policy: StreamIdConflictPolicy::UseExisting as i32, + ..Default::default() + } +} + +fn record(text: &str) -> AppendRecord { + AppendRecord { + body: Some(Payload { + data: text.as_bytes().to_vec(), + ..Default::default() + }), + } +} + +fn millis(ms: u64) -> Option { + let duration = Duration::from_millis(ms); + Some(prost_types::Duration { + seconds: duration.as_secs() as i64, + nanos: duration.subsec_nanos() as i32, + }) +} + +impl Case { + fn stream_ref(&self) -> Option { + Some(StreamRef { + id: self.stream.clone(), + ..Default::default() + }) + } + + async fn create(&self) -> Result { + self.create_with(CreateStreamRequest::default()).await + } + + /// Creates the Stream with the fields of `request`, in the case's namespace and ref. + async fn create_with( + &self, + request: CreateStreamRequest, + ) -> Result { + let request = CreateStreamRequest { + namespace: self.namespace.clone(), + stream_ref: self.stream_ref(), + ..request + }; + self.ext.create_stream(request).await + } + + async fn append( + &self, + producer: Producer, + sequence: i64, + records: Vec, + ) -> Result { + self.ext + .append_records(AppendRecordsRequest { + namespace: self.namespace.clone(), + r#ref: self.stream_ref(), + producer: Some(producer), + sequence, + records, + }) + .await + } + + async fn read(&self, request: ReadRecordsRequest) -> Result { + self.ext + .read_records(ReadRecordsRequest { + namespace: self.namespace.clone(), + r#ref: self.stream_ref(), + ..request + }) + .await + } + + async fn describe(&self) -> Result { + self.ext + .describe_stream(DescribeStreamRequest { + namespace: self.namespace.clone(), + r#ref: self.stream_ref(), + }) + .await + } + + async fn close(&self, producer: Producer) -> Result { + self.ext + .close_stream(CloseStreamRequest { + namespace: self.namespace.clone(), + r#ref: self.stream_ref(), + producer: Some(producer), + }) + .await + } +} + +fn texts(response: &ReadRecordsResponse) -> Vec { + response + .records + .iter() + .map(|record| String::from_utf8(record.body.clone().unwrap().data).unwrap()) + .collect() +} + +/// A create of an open Stream fails as `already_exists`, unless it uses the Stream or it is a +/// retry of the create that made the Stream. Such a retry has the same request id, and it gets +/// `created` true with either policy. A create needs a ref with an id. +async fn a_create_of_an_open_stream_fails_unless_it_uses_it_or_retries() { + for case in cases_for(Needs::CALLS).await { + let first = CreateStreamRequest { + request_id: "create-1".to_string(), + ..Default::default() + }; + assert!(case.create_with(first.clone()).await.unwrap().created); + let other = CreateStreamRequest { + request_id: "create-2".to_string(), + ..Default::default() + }; + for request in [other.clone(), CreateStreamRequest::default()] { + let error = case.create_with(request).await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::AlreadyExists, "stream_already_exists") + ); + } + let used = CreateStreamRequest { + request_id: other.request_id, + ..use_existing() + }; + assert!(!case.create_with(used).await.unwrap().created); + let retry = CreateStreamRequest { + request_id: first.request_id.clone(), + ..use_existing() + }; + for request in [first, retry] { + assert!(case.create_with(request).await.unwrap().created); + } + for stream_ref in [None, Some(StreamRef::default())] { + let request = CreateStreamRequest { + namespace: case.namespace.clone(), + stream_ref, + ..Default::default() + }; + let error = case.ext.create_stream(request).await.unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + } + } +} + +/// A create of a closed Stream fails as closed, before its conflict policy and before a retry of +/// the create that made the Stream. +async fn a_create_of_a_closed_stream_fails() { + for case in cases_for(Needs::CALLS).await { + let first = CreateStreamRequest { + request_id: "create-1".to_string(), + ..Default::default() + }; + case.create_with(first.clone()).await.unwrap(); + case.close(client()).await.unwrap(); + for request in [first, CreateStreamRequest::default()] { + let error = case.create_with(request).await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::FailedPrecondition, "stream_closed") + ); + } + } +} + +/// A close writes no Record, so the describe's last offset stays at the newest Record. A second +/// close of the closed Stream succeeds, and a later append fails as closed. +async fn a_close_writes_no_record_and_a_second_close_succeeds() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let written = case.append(client(), 1, vec![record("a")]).await.unwrap(); + case.close(client()).await.unwrap(); + case.close(client()).await.unwrap(); + let error = case + .append(client(), 2, vec![record("b")]) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_closed"); + let describe = DescribeStreamRequest { + namespace: case.namespace.clone(), + r#ref: case.stream_ref(), + }; + let described = case.ext.describe_stream(describe).await.unwrap(); + assert_eq!(described.status, StreamStatus::Closed as i32); + assert_eq!(described.last_offset, written.last_offset); + } +} + +/// A create of a closed Stream whose Retention passed fails as expired while its Tombstone is +/// there, before the closed check and before `already_exists`. +async fn a_create_of_an_expired_stream_fails() { + for case in short_cases().await { + case.create().await.unwrap(); + case.close(client()).await.unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + let error = case.create().await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::NotFound, "stream_expired") + ); + } +} + +/// A Producer has a kind, a Workflow appends only through a Local Activity, and a Local Activity's +/// epoch names the Workflow Task that dispatched it. +async fn a_producer_has_a_kind_and_is_not_a_workflow() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let no_kind = Producer { + kind: None, + ..client() + }; + let error = case + .append(no_kind, 1, vec![record("x")]) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::InvalidArgument, "stream_invalid") + ); + let error = case.append(workflow("r1"), 1, vec![]).await.unwrap_err(); + assert_eq!(error.code(), Code::InvalidArgument); + let no_task = Producer { + id: "la:r1:call-1".to_string(), + kind: Some(Kind::LocalActivity(LocalActivityProducer { + run_id: "r1".to_string(), + epoch: Some(LocalActivityEpoch { + task: None, + attempt: 1, + }), + })), + }; + let error = case + .append(no_task, 1, vec![record("x")]) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + } +} + +/// A Producer id is opaque bytes. The runtime refuses an empty one and one over the limit, and +/// takes any other, also with separators in it. +async fn a_producer_id_is_opaque_but_not_empty_or_too_long() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + for id in [String::new(), "x".repeat(MAX_PRODUCER_ID_BYTES + 1)] { + let wrong = Producer { id, ..client() }; + let error = case.append(wrong, 1, vec![record("x")]).await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::InvalidArgument, "stream_invalid") + ); + } + let odd = Producer { + id: "ingest|eu:1 é".to_string(), + ..client() + }; + case.append(odd.clone(), 1, vec![record("a")]) + .await + .unwrap(); + case.close(odd).await.unwrap(); + } +} + +/// A close takes a Producer of any kind, an Activity's too. Only an append refuses a Workflow +/// Producer. +async fn a_close_takes_a_producer_of_any_kind() { + for case in cases_for(Needs::UNCHECKED).await { + case.create().await.unwrap(); + let activity = Producer { + id: "act:r1:a:s".to_string(), + kind: Some(Kind::WorkflowActivity(ActivityProducer { + run_id: "r1".to_string(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: Some(prost_types::Timestamp::default()), + attempt: 1, + }), + })), + }; + case.close(activity).await.unwrap(); + assert_eq!( + case.describe().await.unwrap().status, + StreamStatus::Closed as i32 + ); + } +} + +/// An append and a close of a Producer that lacks a required field fail as invalid before the +/// Store, so the Stream gets no Record and stays open. +async fn a_write_with_a_missing_required_field_is_invalid() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let no_epoch = Producer { + id: "la:r1:call-1".to_string(), + kind: Some(Kind::LocalActivity(LocalActivityProducer { + run_id: "r1".to_string(), + epoch: None, + })), + }; + let error = case + .append(no_epoch.clone(), 1, vec![record("x")]) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::InvalidArgument, "stream_invalid") + ); + let error = case.close(no_epoch).await.unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + // A Workflow Producer may close with no epoch, but one that it sends must order. + let mut no_task = workflow("r1"); + let Some(Kind::Workflow(kind)) = no_task.kind.as_mut() else { + unreachable!() + }; + kind.epoch = Some(WorkflowTaskEpoch::default()); + let error = case.close(no_task).await.unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Open as i32); + assert!(describe.last_offset.is_empty()); + } +} + +/// An append with a Record that has no body fails as invalid before the Store, so no Record of the +/// batch is written. +async fn an_append_with_a_record_that_has_no_body_is_invalid() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let records = vec![record("a"), AppendRecord::default()]; + let error = case.append(client(), 1, records).await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::InvalidArgument, "stream_invalid") + ); + assert!(case.describe().await.unwrap().last_offset.is_empty()); + } +} + +/// An append with no sequence takes the next one of its Producer's epoch, as a call with no writer +/// state sends it. Nothing dedupes it, so its retry writes the batch again. A negative sequence +/// fails. +async fn an_append_with_no_sequence_takes_the_next_one_of_its_epoch() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let first = case.append(client(), 0, vec![record("a")]).await.unwrap(); + let retry = case.append(client(), 0, vec![record("a")]).await.unwrap(); + assert!( + retry.first_offset > first.last_offset, + "{first:?} {retry:?}" + ); + let resend = case.append(client(), 2, vec![record("a")]).await.unwrap(); + assert_eq!(resend, retry); + let error = case + .append(client(), -1, vec![record("b")]) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + } +} + +/// A read with no offset starts at the beginning, and a read is exclusive from its offset. A read +/// after the newest offset gives no Record, so a describe and a read give only new Records. An +/// offset past the newest fails with `stream_offset_too_high`, and one that decodes below zero or +/// has more than 8 bytes is invalid. +async fn a_read_is_exclusive_from_its_offset_and_get_offset_reads_only_new_records() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let first = case + .append(client(), 1, vec![record("1"), record("2")]) + .await + .unwrap(); + let all = case.read(ReadRecordsRequest::default()).await.unwrap(); + assert_eq!(texts(&all), ["1", "2"]); + assert_eq!(all.records[0].offset, first.first_offset); + assert_eq!(all.last_offset, first.last_offset); + let from = case + .read(ReadRecordsRequest { + from_offset: first.first_offset.clone(), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(texts(&from), ["2"]); + let newest = case.describe().await.unwrap().last_offset; + let none = case + .read(ReadRecordsRequest { + from_offset: newest.clone(), + ..Default::default() + }) + .await + .unwrap(); + assert!(none.records.is_empty()); + assert_eq!(none.last_offset, newest); + let error = case + .read(ReadRecordsRequest { + from_offset: wire(off(&newest) + 1), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::OutOfRange, "stream_offset_too_high") + ); + let error = case + .read(ReadRecordsRequest { + // An 8-byte value with the top bit set decodes to a negative offset. + from_offset: vec![0x80, 0, 0, 0, 0, 0, 0, 0], + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + let error = case + .read(ReadRecordsRequest { + from_offset: vec![0; 9], + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + // The largest offset has no next one, and it is past the newest Record too. + let error = case + .read(ReadRecordsRequest { + from_offset: wire(i64::MAX), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_offset_too_high"); + } +} + +/// A read of a Stream that nobody created fails at once with `stream_not_found`, also with a long +/// wait. +async fn a_read_of_an_uncreated_stream_fails_at_once() { + for case in cases_for(Needs::CALLS).await { + let started = Instant::now(); + let error = case + .read(ReadRecordsRequest { + wait: millis(5000), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::NotFound, "stream_not_found") + ); + assert!(started.elapsed() < Duration::from_secs(2)); + } +} + +/// A quiet read ends empty after its wait, and a long poll wakes on an append. +async fn a_quiet_read_ends_empty_after_its_wait_and_a_long_poll_wakes_on_append() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let first = case.append(client(), 1, vec![record("1")]).await.unwrap(); + let from = first.last_offset; + let started = Instant::now(); + let quiet = case + .read(ReadRecordsRequest { + from_offset: from.clone(), + wait: millis(200), + ..Default::default() + }) + .await + .unwrap(); + assert!(started.elapsed() >= Duration::from_millis(200)); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(quiet.records.is_empty()); + assert_eq!(quiet.last_offset, from); + assert!(!quiet.done); + let ext = case.ext.clone(); + let (namespace, stream) = (case.namespace.clone(), case.stream.clone()); + let writer = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(150)).await; + let request = AppendRecordsRequest { + namespace, + r#ref: Some(StreamRef { + id: stream, + ..Default::default() + }), + producer: Some(client()), + sequence: 2, + records: vec![record("2")], + }; + ext.append_records(request).await + }); + let woke = case + .read(ReadRecordsRequest { + from_offset: from, + wait: millis(5000), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(texts(&woke), ["2"]); + writer.await.unwrap().unwrap(); + } +} + +/// A read that gives the last Record of a closed Stream is done, and so is every later read, which +/// gives no Record. A close refuses later appends. +async fn a_close_ends_every_read_with_done() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + case.append(client(), 1, vec![record("1")]).await.unwrap(); + case.close(client()).await.unwrap(); + let read = case.read(ReadRecordsRequest::default()).await.unwrap(); + assert_eq!(texts(&read), ["1"]); + assert!(read.done); + let again = case + .read(ReadRecordsRequest { + from_offset: read.last_offset.clone(), + ..Default::default() + }) + .await + .unwrap(); + assert!(again.records.is_empty() && again.done); + assert_eq!(again.last_offset, read.last_offset); + assert_eq!( + case.describe().await.unwrap().status, + StreamStatus::Closed as i32 + ); + let error = case + .append(client(), 2, vec![record("2")]) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::FailedPrecondition, "stream_closed") + ); + } +} + +/// A read of a closed Stream whose Retention passed fails as expired, from any offset, instead +/// of waiting out its wait. +async fn a_read_of_an_expired_closed_stream_fails_instead_of_waiting() { + for case in short_cases().await { + case.create().await.unwrap(); + let first = case.append(client(), 1, vec![record("1")]).await.unwrap(); + case.close(client()).await.unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Expired as i32); + for request in [ + ReadRecordsRequest::default(), + ReadRecordsRequest { + from_offset: first.first_offset, + ..Default::default() + }, + ReadRecordsRequest { + from_offset: describe.last_offset, + ..Default::default() + }, + ] { + let started = Instant::now(); + let error = case + .read(ReadRecordsRequest { + wait: millis(1500), + ..request + }) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_expired"); + assert!(started.elapsed() < Duration::from_secs(1)); + } + } +} + +/// An open Stream keeps its Records past its Retention, which counts from the close. So a read +/// gets every Record until one Retention after the close, and then the Stream is expired. The +/// Retention is the Store's, since a create takes none. +async fn the_retention_counts_from_the_close() { + for case in short_cases().await { + case.create().await.unwrap(); + let first = case + .append(client(), 1, vec![record("a"), record("b")]) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + let after_first = ReadRecordsRequest { + from_offset: first.first_offset.clone(), + ..Default::default() + }; + let read = case.read(after_first.clone()).await.unwrap(); + assert_eq!(texts(&read), ["b"]); + case.append(client(), 2, vec![record("c")]).await.unwrap(); + case.close(client()).await.unwrap(); + let read = case.read(ReadRecordsRequest::default()).await.unwrap(); + assert_eq!(texts(&read), ["a", "b", "c"]); + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Closed as i32); + tokio::time::sleep(Duration::from_millis(400)).await; + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Expired as i32); + let error = case.read(after_first).await.unwrap_err(); + assert_eq!(reason(&error), "stream_expired"); + } +} + +/// With a last-write TTL in the Store config, an open Stream expires that long after its last +/// write. Each write starts the TTL again. +async fn an_open_stream_expires_its_last_write_ttl_after_its_last_write() { + let config = RuntimeConfig { + last_write_ttl: Some(Duration::from_millis(500)), + ..Default::default() + }; + for case in cases_with(Needs::SHORT_TIMES, &config).await { + case.create().await.unwrap(); + case.append(client(), 1, vec![record("a")]).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + case.append(client(), 2, vec![record("b")]).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Open as i32); + tokio::time::sleep(Duration::from_millis(500)).await; + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Expired as i32); + let error = case + .append(client(), 3, vec![record("c")]) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_expired"); + } +} + +/// A create answer names the configured Store, and the Store never reads the name. +async fn a_create_answer_names_the_store_and_the_store_never_reads_it() { + for ext in super::support::named_runtimes("redis-a") { + let unique = unique(); + let case = Case { + ext, + namespace: format!("ns-{unique}"), + stream: format!("s-{unique}"), + }; + let created = case.create().await.unwrap(); + let stream = created.r#ref.unwrap(); + assert_eq!(stream.id, case.stream); + assert_eq!(stream.store, "redis-a"); + // A ref with any Store name, or none, reaches the same Stream. + let named = |store: &str| { + Some(StreamRef { + id: case.stream.clone(), + store: store.to_string(), + }) + }; + for (sequence, stream) in [(1, named("redis-a")), (2, named("")), (3, named("other"))] { + let request = AppendRecordsRequest { + namespace: case.namespace.clone(), + r#ref: stream, + producer: Some(client()), + sequence, + records: vec![record(&sequence.to_string())], + }; + case.ext.append_records(request).await.unwrap(); + } + let read = ReadRecordsRequest { + namespace: case.namespace.clone(), + r#ref: named("other"), + ..Default::default() + }; + let read = case.ext.read_records(read).await.unwrap(); + assert_eq!(texts(&read), ["1", "2", "3"]); + } + // With no Store name in the config, the ref has none, and lang reads it as the default. + for case in cases_for(Needs::OWN_CONFIG).await { + let created = case.create().await.unwrap(); + assert_eq!(created.r#ref.unwrap().store, ""); + } +} + +/// A read of a closed Stream gives `done` only with its last Record. With no Record left, a read +/// answers at once with `done`, even with a long wait. +async fn a_read_of_a_closed_stream_with_no_record_left_answers_at_once() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let records = vec![record("1"), record("2")]; + let written = case.append(client(), 1, records).await.unwrap(); + case.close(client()).await.unwrap(); + let first = case + .read(ReadRecordsRequest { + max_records: 1, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(texts(&first), ["1"]); + assert!(!first.done); + let rest = case + .read(ReadRecordsRequest { + from_offset: first.last_offset, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(texts(&rest), ["2"]); + assert!(rest.done); + let started = Instant::now(); + let read = case + .read(ReadRecordsRequest { + from_offset: written.last_offset.clone(), + wait: millis(5000), + ..Default::default() + }) + .await + .unwrap(); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(read.records.is_empty() && read.done); + assert_eq!(read.last_offset, written.last_offset); + } +} + +/// A long poll at the end of an open Stream wakes on its close and answers with `done`. +async fn a_waiting_read_wakes_with_done_on_a_close() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let written = case.append(client(), 1, vec![record("1")]).await.unwrap(); + let ext = case.ext.clone(); + let close = CloseStreamRequest { + namespace: case.namespace.clone(), + r#ref: case.stream_ref(), + producer: Some(client()), + }; + let closer = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(150)).await; + ext.close_stream(close).await + }); + let started = Instant::now(); + let woke = case + .read(ReadRecordsRequest { + from_offset: written.last_offset, + wait: millis(5000), + ..Default::default() + }) + .await + .unwrap(); + assert!(started.elapsed() < Duration::from_secs(4)); + assert!(woke.records.is_empty() && woke.done); + closer.await.unwrap().unwrap(); + } +} + +/// A page stays under `max_bytes`, and a larger Record is still returned alone. +async fn a_page_keeps_under_max_bytes_and_a_larger_record_comes_alone() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let big = "x".repeat(1000); + let records = vec![record("a"), record("b"), record(&big), record("c")]; + case.append(client(), 1, records).await.unwrap(); + let capped = |from_offset| ReadRecordsRequest { + from_offset, + max_bytes: 200, + ..Default::default() + }; + let first = case.read(capped(vec![])).await.unwrap(); + assert_eq!(texts(&first), ["a", "b"]); + let second = case.read(capped(first.last_offset)).await.unwrap(); + assert_eq!(texts(&second), [big]); + let third = case.read(capped(second.last_offset)).await.unwrap(); + assert_eq!(texts(&third), ["c"]); + } +} + +/// The cases of the suite, in file order. +pub const CASES: &[super::Case] = &[ + case!(a_create_of_an_open_stream_fails_unless_it_uses_it_or_retries), + case!(a_create_of_a_closed_stream_fails), + case!(a_close_writes_no_record_and_a_second_close_succeeds), + case!(a_create_of_an_expired_stream_fails), + case!(a_producer_has_a_kind_and_is_not_a_workflow), + case!(a_producer_id_is_opaque_but_not_empty_or_too_long), + case!(a_close_takes_a_producer_of_any_kind), + case!(a_write_with_a_missing_required_field_is_invalid), + case!(an_append_with_a_record_that_has_no_body_is_invalid), + case!(an_append_with_no_sequence_takes_the_next_one_of_its_epoch), + case!(a_read_is_exclusive_from_its_offset_and_get_offset_reads_only_new_records), + case!(a_read_of_an_uncreated_stream_fails_at_once), + case!(a_quiet_read_ends_empty_after_its_wait_and_a_long_poll_wakes_on_append), + case!(a_close_ends_every_read_with_done), + case!(a_read_of_an_expired_closed_stream_fails_instead_of_waiting), + case!(the_retention_counts_from_the_close), + case!(an_open_stream_expires_its_last_write_ttl_after_its_last_write), + case!(a_create_answer_names_the_store_and_the_store_never_reads_it), + case!(a_read_of_a_closed_stream_with_no_record_left_answers_at_once), + case!(a_waiting_read_wakes_with_done_on_a_close), + case!(a_page_keeps_under_max_bytes_and_a_larger_record_comes_alone), +]; diff --git a/crates/streams/src/testing/support.rs b/crates/streams/src/testing/support.rs index 38bfbc1d3..2ae17d5d5 100644 --- a/crates/streams/src/testing/support.rs +++ b/crates/streams/src/testing/support.rs @@ -3,9 +3,9 @@ #![allow(dead_code)] -use super::{RawKeys, test_stores}; +use super::{RawKeys, RuntimeConfig, test_stores}; use crate::{ - StreamResult, + StreamResult, StreamRuntime, digest::batch_digest, proto::{ AppendRecord, LocalActivityEpoch, LocalActivityProducer, Producer, StreamRecord, StreamRef, @@ -272,3 +272,38 @@ impl Target { self.store.close(&self.stream_ref(), producer).await } } + +/// One runtime under test, the namespace its cases use, and what its Store gives. +pub(crate) struct Runtime { + pub(crate) ext: Arc, + /// The namespace of a Store that has fixed ones, such as the server. `None` lets the case + /// make one of its own. + pub(crate) namespace: Option, + pub(crate) gives: Needs, +} + +/// A runtime with `config` on each Store under test that gives `needs`. +pub(crate) async fn runtimes(needs: Needs, config: &RuntimeConfig) -> Vec { + test_stores() + .into_iter() + .filter(|test_store| test_store.gives().gives(needs)) + .map(|test_store| Runtime { + ext: test_store.runtime(config), + namespace: None, + gives: test_store.gives(), + }) + .collect() +} + +/// A runtime on each Store under test, whose refs carry `store_name`. +pub(crate) fn named_runtimes(store_name: &str) -> Vec> { + test_stores() + .into_iter() + .map(|test_store| { + test_store.runtime(&RuntimeConfig { + store_name: store_name.to_string(), + ..Default::default() + }) + }) + .collect() +} diff --git a/crates/streams/tests/capabilities.rs b/crates/streams/tests/capabilities.rs new file mode 100644 index 000000000..addf849ed --- /dev/null +++ b/crates/streams/tests/capabilities.rs @@ -0,0 +1,213 @@ +//! A Store that lacks a capability refuses every call that needs it, by name, and writes nothing. +//! The tests use the memory Store with one capability masked. + +use prost::Message; +use std::{ + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use temporalio_common::protos::temporal::api::common::v1::Payload; +use temporalio_streams::{ + StreamError, StreamResult, StreamRuntime, + memory::MemoryStore, + proto::{ + AppendRecord, AppendRecordsRequest, ClientProducer, CloseStreamRequest, + CreateStreamRequest, DescribeStreamRequest, DescribeStreamResponse, Producer, + StreamErrorDetails, StreamErrorReason, StreamRef, StreamStatus, producer::Kind, + }, + store::{Appended, Batch, Capabilities, Closed, Create, Created, Entry, Store, StreamState}, +}; +use tokio::time::Instant; +use tonic::Code; + +/// A memory Store that lacks the capabilities in `without`, as a Store with fewer would. +struct Masked { + inner: Arc, + without: Capabilities, +} + +#[async_trait::async_trait] +impl Store for Masked { + fn capabilities(&self) -> Capabilities { + self.inner.capabilities().without(self.without) + } + + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult { + self.inner.create(stream_ref, create).await + } + + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult { + self.inner.append(stream_ref, batch).await + } + + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)> { + self.inner.read(stream_ref, from, count).await + } + + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult { + self.inner.wait_after(stream_ref, after, deadline).await + } + + // The masked calls still work underneath, so a call that skips the check would succeed, + // and the tests would see it. + async fn close(&self, stream_ref: &StreamRef, producer: &Producer) -> StreamResult { + self.inner.close(stream_ref, producer).await + } +} + +const NAMESPACE: &str = "ns"; + +struct Case { + ext: Arc, + memory: Arc, + stream: String, +} + +fn case(without: Capabilities) -> Case { + let memory = Arc::new(MemoryStore::new( + Duration::from_secs(3600), + Some(Duration::from_secs(7200)), + )); + let store = Arc::new(Masked { + inner: memory.clone(), + without, + }); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + Case { + ext: StreamRuntime::new("", store, None), + memory, + stream: format!("wf:run-{unique}"), + } +} + +fn record() -> AppendRecord { + AppendRecord { + body: Some(Payload::default()), + } +} + +fn client() -> Producer { + Producer { + id: "client:me:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } +} + +/// The capability an error names, after it checked the code and the reason. +fn refused_capability(error: &StreamError) -> String { + assert_eq!(error.code(), Code::Unimplemented, "{error:?}"); + let details = StreamErrorDetails::decode(error.details().as_slice()).unwrap(); + assert_eq!(details.reason(), StreamErrorReason::Unsupported); + error + .message + .rsplit(' ') + .next() + .unwrap_or_default() + .to_string() +} + +impl Case { + fn stream_ref(&self) -> Option { + Some(StreamRef { + id: self.stream.clone(), + ..Default::default() + }) + } + + async fn call( + &self, + operation: &str, + request: Req, + ) -> Result, StreamError> { + self.ext.call(operation, &request.encode_to_vec()).await + } + + async fn create(&self) { + let request = CreateStreamRequest { + namespace: NAMESPACE.to_string(), + stream_ref: self.stream_ref(), + ..Default::default() + }; + self.call("CreateStream", request).await.unwrap(); + } + + fn log_len(&self) -> usize { + self.memory.test_log_len(&self.stream) + } +} + +#[tokio::test] +async fn an_append_without_the_fence_is_refused_with_nothing_written() { + let case = case(Capabilities::PRODUCER_FENCE); + case.create().await; + let request = AppendRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: case.stream_ref(), + producer: Some(client()), + sequence: 1, + records: vec![record()], + }; + let error = case.call("AppendRecords", request).await.unwrap_err(); + assert_eq!(refused_capability(&error), "producer_fence"); + assert_eq!(case.log_len(), 0); +} + +#[tokio::test] +async fn an_append_with_no_sequence_without_the_capability_is_refused_with_nothing_written() { + let case = case(Capabilities::NEXT_SEQUENCE); + case.create().await; + let request = AppendRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: case.stream_ref(), + producer: Some(client()), + records: vec![record()], + ..Default::default() + }; + let error = case + .call("AppendRecords", request.clone()) + .await + .unwrap_err(); + assert_eq!(refused_capability(&error), "next_sequence"); + assert_eq!(case.log_len(), 0); + // A batch with its own sequence needs no number from the Store. + let numbered = AppendRecordsRequest { + sequence: 1, + ..request + }; + case.call("AppendRecords", numbered).await.unwrap(); + assert_eq!(case.log_len(), 1); +} + +#[tokio::test] +async fn a_close_without_the_capability_is_refused_and_the_stream_stays_open() { + let case = case(Capabilities::CLOSE); + case.create().await; + let request = CloseStreamRequest { + namespace: NAMESPACE.to_string(), + r#ref: case.stream_ref(), + producer: Some(client()), + }; + let error = case.call("CloseStream", request).await.unwrap_err(); + assert_eq!(refused_capability(&error), "close"); + assert_eq!(case.log_len(), 0); + let describe = DescribeStreamRequest { + namespace: NAMESPACE.to_string(), + r#ref: case.stream_ref(), + }; + let bytes = case.call("DescribeStream", describe).await.unwrap(); + let described = DescribeStreamResponse::decode(bytes.as_slice()).unwrap(); + assert_eq!(described.status, StreamStatus::Open as i32); +} diff --git a/crates/streams/tests/scans.rs b/crates/streams/tests/scans.rs new file mode 100644 index 000000000..f1a52610b --- /dev/null +++ b/crates/streams/tests/scans.rs @@ -0,0 +1,206 @@ +//! How a read scans the log, seen through a memory Store that can trim under a read. + +use prost::Message; +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; +use temporalio_common::protos::temporal::api::common::v1::Payload; +use temporalio_streams::{ + StreamError, StreamResult, StreamRuntime, + memory::MemoryStore, + proto::{ + AppendRecord, AppendRecordsRequest, AppendRecordsResponse, ClientProducer, + CreateStreamRequest, CreateStreamResponse, Producer, ReadRecordsRequest, + ReadRecordsResponse, StreamErrorDetails, StreamErrorReason, StreamRef, producer::Kind, + }, + store::{Appended, Batch, Capabilities, Create, Created, Entry, Store, StreamState}, +}; +use tokio::time::Instant; + +const NAMESPACE: &str = "ns"; +const STREAM: &str = "s"; + +/// The memory Store. Once armed, its next read that asks for log entries finds its first entry +/// trimmed, as a trim just before that read leaves it. It counts the entries that reads return. +struct Trimming { + inner: MemoryStore, + trim_next: AtomicBool, + pulled: AtomicUsize, +} + +#[async_trait::async_trait] +impl Store for Trimming { + fn capabilities(&self) -> Capabilities { + self.inner.capabilities() + } + + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult { + self.inner.create(stream_ref, create).await + } + + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult { + self.inner.append(stream_ref, batch).await + } + + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)> { + let (mut state, mut entries) = self.inner.read(stream_ref, from, count).await?; + if count > 0 && !entries.is_empty() && self.trim_next.swap(false, Ordering::Relaxed) { + state.trimmed = entries.remove(0).offset; + } + self.pulled.fetch_add(entries.len(), Ordering::Relaxed); + Ok((state, entries)) + } + + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult { + self.inner.wait_after(stream_ref, after, deadline).await + } +} + +fn trimming() -> (Arc, Arc) { + let store = Arc::new(Trimming { + inner: MemoryStore::new(Duration::from_secs(3600), Some(Duration::from_secs(7200))), + trim_next: AtomicBool::new(false), + pulled: AtomicUsize::new(0), + }); + let ext = StreamRuntime::new("", store.clone(), None); + (store, ext) +} + +async fn call( + ext: &StreamRuntime, + operation: &str, + request: Req, +) -> Resp { + try_call(ext, operation, request).await.unwrap() +} + +async fn try_call( + ext: &StreamRuntime, + operation: &str, + request: Req, +) -> Result { + let bytes = ext.call(operation, &request.encode_to_vec()).await?; + Ok(Resp::decode(bytes.as_slice()).unwrap()) +} + +async fn create(ext: &StreamRuntime) { + let _: CreateStreamResponse = call( + ext, + "CreateStream", + CreateStreamRequest { + namespace: NAMESPACE.to_string(), + stream_ref: stream_ref(), + ..Default::default() + }, + ) + .await; +} + +fn client() -> Producer { + Producer { + id: "client:me:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } +} + +fn stream_ref() -> Option { + Some(StreamRef { + id: STREAM.to_string(), + ..Default::default() + }) +} + +async fn append(ext: &StreamRuntime, sequence: i64, count: usize) -> AppendRecordsResponse { + append_sized(ext, sequence, count, 0).await +} + +/// Appends `count` Records, each with `size` bytes after its number. +async fn append_sized( + ext: &StreamRuntime, + sequence: i64, + count: usize, + size: usize, +) -> AppendRecordsResponse { + let records = (0..count) + .map(|i| AppendRecord { + body: Some(Payload { + data: format!("{i}{}", "x".repeat(size)).into_bytes(), + ..Default::default() + }), + }) + .collect(); + call( + ext, + "AppendRecords", + AppendRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: stream_ref(), + producer: Some(client()), + sequence, + records, + }, + ) + .await +} + +/// A trim between a read's state and its log read fails a read from an offset as expired, so that +/// the Reader always sees the gap. A read from the start skips the trimmed Records. +#[tokio::test] +async fn a_trim_under_a_read_from_an_offset_fails_it_as_expired() { + let (store, ext) = trimming(); + create(&ext).await; + let first = append(&ext, 1, 3).await; + let read = |from_offset| ReadRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: stream_ref(), + from_offset, + ..Default::default() + }; + store.trim_next.store(true, Ordering::Relaxed); + let error = try_call::<_, ReadRecordsResponse>(&ext, "ReadRecords", read(first.first_offset)) + .await + .unwrap_err(); + let details = StreamErrorDetails::decode(error.details().as_slice()).unwrap(); + assert_eq!(details.reason(), StreamErrorReason::Expired); + store.trim_next.store(true, Ordering::Relaxed); + let from_start: ReadRecordsResponse = call(&ext, "ReadRecords", read(vec![])).await; + assert_eq!(from_start.records.len(), 2); +} + +/// A read with a byte cap reads about the entries that its page holds from the Store, so a small +/// cap on large Records doesn't read a whole page of them. +#[tokio::test] +async fn a_capped_read_pulls_about_its_page_from_the_store() { + let (store, ext) = trimming(); + create(&ext).await; + append_sized(&ext, 1, 50, 1000).await; + store.pulled.store(0, Ordering::Relaxed); + let page: ReadRecordsResponse = call( + &ext, + "ReadRecords", + ReadRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: stream_ref(), + max_bytes: 2500, + ..Default::default() + }, + ) + .await; + assert_eq!(page.records.len(), 2); + let pulled = store.pulled.load(Ordering::Relaxed); + assert!(pulled <= 4, "{pulled}"); +} From 4bb74a0d9a181017f2a8dfb056e21d480cd4bfa3 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Thu, 12 Nov 2026 22:16:21 -0700 Subject: [PATCH 4/5] Tested the Producer rules that the id check leaves to the caller. The fence of a Producer is kept by kind and id, so the id of a run-based kind must hold its run. The shared suites now cover two runs, two kinds with one id, any text in an id and a close of each kind, so every Store keeps these rules. --- crates/streams/README.md | 7 +- crates/streams/src/epoch.rs | 12 +-- crates/streams/src/ids.rs | 13 ++- crates/streams/src/testing/service.rs | 111 ++++++++++++++++++++------ crates/streams/src/testing/store.rs | 68 +++++++++++++++- crates/streams/src/testing/support.rs | 8 -- 6 files changed, 169 insertions(+), 50 deletions(-) diff --git a/crates/streams/README.md b/crates/streams/README.md index c4561e3d1..9c1bd985d 100644 --- a/crates/streams/README.md +++ b/crates/streams/README.md @@ -19,8 +19,11 @@ re-exports them, so a value needs no conversion between the crates. each file. Each Store reads `fields.json` through the shared Store suite. `digests.json` is the golden of the crate's batch digest. -A Stream id and a Producer id are opaque bytes. The caller builds both. The runtime refuses an -empty id of either kind, and one over 4096 bytes. +A Stream id and a Producer id are opaque text. The caller builds both. The runtime refuses an +empty id of either kind, and one over 4096 bytes. A Producer id names one Producer in one Stream. +A Store keeps the fence of a Producer by kind and id, and an epoch has no run. So the id of a +run-based kind must hold its run id. A caller that uses one id in two runs gets `FENCED` in the +second run when that run's epoch is lower. ## The runtime diff --git a/crates/streams/src/epoch.rs b/crates/streams/src/epoch.rs index 63378e2ac..d8babde68 100644 --- a/crates/streams/src/epoch.rs +++ b/crates/streams/src/epoch.rs @@ -9,7 +9,6 @@ use crate::{ proto::{Producer, WorkflowTaskEpoch, producer::Kind}, }; use prost_types::Timestamp; -use std::cmp::Ordering; /// The kind of a Producer, which is the case of its epoch. The stored field names and keys hold /// its number, so each kind keeps its number. @@ -120,19 +119,14 @@ pub fn task_of(kind: Option<&Kind>) -> WorkflowTaskEpoch { } } -/// Orders two epochs field by field. -pub fn compare_epochs(a: Option<&Kind>, b: Option<&Kind>) -> Ordering { - epoch_key(a).cmp(&epoch_key(b)) -} - #[cfg(test)] mod tests { use super::*; - use crate::proto::{LocalActivityEpoch, LocalActivityProducer, producer::Kind}; + use crate::proto::{LocalActivityEpoch, LocalActivityProducer}; // Without the clamp, the bytes of a negative field would outrank every real epoch. #[test] - fn a_negative_field_compares_as_zero() { + fn a_negative_field_keys_as_zero() { let negative = Kind::LocalActivity(LocalActivityProducer { epoch: Some(LocalActivityEpoch { task: None, @@ -140,6 +134,6 @@ mod tests { }), ..Default::default() }); - assert_eq!(compare_epochs(Some(&negative), None), Ordering::Equal); + assert_eq!(epoch_key(Some(&negative)), epoch_key(None)); } } diff --git a/crates/streams/src/ids.rs b/crates/streams/src/ids.rs index 0f7dbf172..432e1e4c6 100644 --- a/crates/streams/src/ids.rs +++ b/crates/streams/src/ids.rs @@ -1,5 +1,5 @@ //! The checks of Stream ids and Producers, and the Redis key layout. A caller builds both kinds of -//! id, so the crate treats each as opaque bytes. `testdata/ids.json` has the shared vectors of the +//! id, so the crate treats each as opaque text. `testdata/ids.json` has the shared vectors of the //! key layout. //! //! ```text @@ -23,7 +23,9 @@ pub const MAX_STREAM_ID_BYTES: usize = 4096; /// The longest Producer id, in bytes. A caller builds the id from names that Temporal limits to /// 1000 characters, so an id with two of them still fits. A Store keeps the id in one meta field of -/// each Producer for as long as the Stream lives, and in the first entry of each of its batches. +/// each Producer for as long as the Stream lives. Each Record of a batch holds its Producer, so the +/// size check counts the Producer once for each Record against +/// [MAX_BATCH_BYTES](crate::store::MAX_BATCH_BYTES). pub const MAX_PRODUCER_ID_BYTES: usize = 4096; /// The key prefix when the Store options set none. @@ -56,8 +58,11 @@ pub fn check_stream_id(id: &str) -> StreamResult<()> { Ok(()) } -/// Refuses a Producer with no id, an id over [MAX_PRODUCER_ID_BYTES], or no kind. The id is -/// opaque, so the Store requires no shape of it. +/// Refuses a Producer with no id, an id over [MAX_PRODUCER_ID_BYTES], or no kind. The id is opaque +/// text, so the Store requires no shape of it. It names one Producer in one Stream. A Store keeps +/// the fence of a Producer by kind and id, and an epoch has no run. So the id of a run-based kind +/// must hold its run id. A caller that uses one id in two runs gets `FENCED` in the second run when +/// that run's epoch is lower. pub fn check_producer(producer: &Producer) -> StreamResult<()> { if producer.id.is_empty() { return Err(StreamError::invalid("The Producer id is empty")); diff --git a/crates/streams/src/testing/service.rs b/crates/streams/src/testing/service.rs index 393cc9bce..4fa5a3b08 100644 --- a/crates/streams/src/testing/service.rs +++ b/crates/streams/src/testing/service.rs @@ -358,7 +358,7 @@ async fn a_producer_has_a_kind_and_is_not_a_workflow() { } } -/// A Producer id is opaque bytes. The runtime refuses an empty one and one over the limit, and +/// A Producer id is opaque text. The runtime refuses an empty one and one over the limit, and /// takes any other, also with separators in it. async fn a_producer_id_is_opaque_but_not_empty_or_too_long() { for case in cases_for(Needs::CALLS).await { @@ -382,26 +382,90 @@ async fn a_producer_id_is_opaque_but_not_empty_or_too_long() { } } -/// A close takes a Producer of any kind, an Activity's too. Only an append refuses a Workflow -/// Producer. -async fn a_close_takes_a_producer_of_any_kind() { - for case in cases_for(Needs::UNCHECKED).await { +/// A Producer id may hold any text, such as a NUL, a newline, the `|` and `:` that the Store +/// fields use, and non-ASCII text. An id of the longest length is taken too. A read gives each +/// Record with the id as sent. +async fn a_producer_id_of_any_text_or_the_longest_length_reads_back() { + for case in cases_for(Needs::CALLS).await { case.create().await.unwrap(); - let activity = Producer { - id: "act:r1:a:s".to_string(), - kind: Some(Kind::WorkflowActivity(ActivityProducer { - run_id: "r1".to_string(), - epoch: Some(ActivityEpoch { - attempt_scheduled_time: Some(prost_types::Timestamp::default()), + let ids = [ + "nul\0line\nbar|colon:é".to_string(), + "p".repeat(MAX_PRODUCER_ID_BYTES), + ]; + for (sequence, id) in ids.iter().enumerate() { + let producer = Producer { + id: id.clone(), + ..client() + }; + case.append(producer, 1, vec![record(&sequence.to_string())]) + .await + .unwrap(); + } + let read = case.read(ReadRecordsRequest::default()).await.unwrap(); + assert_eq!(texts(&read), ["0", "1"]); + let read_ids: Vec<_> = read + .records + .iter() + .map(|record| record.producer.as_ref().unwrap().id.clone()) + .collect(); + assert_eq!(read_ids, ids); + } +} + +/// A close takes a Producer of any kind. The Workflow Producer sends no epoch, as its close on a +/// Store does. Only an append refuses a Workflow Producer. +async fn a_close_takes_a_producer_of_any_kind() { + let activity = ActivityProducer { + run_id: "r1".to_string(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: Some(prost_types::Timestamp::default()), + attempt: 1, + }), + }; + let kinds = [ + Kind::Workflow(WorkflowProducer { + run_id: "r1".to_string(), + epoch: None, + }), + Kind::LocalActivity(LocalActivityProducer { + run_id: "r1".to_string(), + epoch: Some(LocalActivityEpoch { + task: Some(WorkflowTaskEpoch { + started_event_id: 3, attempt: 1, + started_time: None, }), - })), - }; - case.close(activity).await.unwrap(); - assert_eq!( - case.describe().await.unwrap().status, - StreamStatus::Closed as i32 - ); + attempt: 1, + }), + }), + Kind::WorkflowActivity(activity.clone()), + Kind::StandaloneActivity(ActivityProducer { + run_id: String::new(), + ..activity + }), + Kind::Client(ClientProducer {}), + ]; + for case in cases_for(Needs::UNCHECKED).await { + for (at, kind) in kinds.iter().enumerate() { + let case = Case { + ext: case.ext.clone(), + namespace: case.namespace.clone(), + stream: format!("{}-{at}", case.stream), + }; + case.create().await.unwrap(); + let producer = Producer { + id: format!("p:r1:{at}"), + kind: Some(kind.clone()), + }; + case.close(producer.clone()) + .await + .unwrap_or_else(|error| panic!("{producer:?} {error:?}")); + assert_eq!( + case.describe().await.unwrap().status, + StreamStatus::Closed as i32, + "{producer:?}" + ); + } } } @@ -456,9 +520,9 @@ async fn an_append_with_a_record_that_has_no_body_is_invalid() { } } -/// An append with no sequence takes the next one of its Producer's epoch, as a call with no writer -/// state sends it. Nothing dedupes it, so its retry writes the batch again. A negative sequence -/// fails. +/// An append with no sequence takes the next one of its Producer's epoch, as a caller that tracks +/// no sequence sends it. Nothing dedupes it, so its retry writes the batch again. A negative +/// sequence fails. async fn an_append_with_no_sequence_takes_the_next_one_of_its_epoch() { for case in cases_for(Needs::CALLS).await { case.create().await.unwrap(); @@ -593,7 +657,7 @@ async fn a_quiet_read_ends_empty_after_its_wait_and_a_long_poll_wakes_on_append( assert!(!quiet.done); let ext = case.ext.clone(); let (namespace, stream) = (case.namespace.clone(), case.stream.clone()); - let writer = tokio::spawn(async move { + let appender = tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(150)).await; let request = AppendRecordsRequest { namespace, @@ -616,7 +680,7 @@ async fn a_quiet_read_ends_empty_after_its_wait_and_a_long_poll_wakes_on_append( .await .unwrap(); assert_eq!(texts(&woke), ["2"]); - writer.await.unwrap().unwrap(); + appender.await.unwrap().unwrap(); } } @@ -891,6 +955,7 @@ pub const CASES: &[super::Case] = &[ case!(a_create_of_an_expired_stream_fails), case!(a_producer_has_a_kind_and_is_not_a_workflow), case!(a_producer_id_is_opaque_but_not_empty_or_too_long), + case!(a_producer_id_of_any_text_or_the_longest_length_reads_back), case!(a_close_takes_a_producer_of_any_kind), case!(a_write_with_a_missing_required_field_is_invalid), case!(an_append_with_a_record_that_has_no_body_is_invalid), diff --git a/crates/streams/src/testing/store.rs b/crates/streams/src/testing/store.rs index 5aea896a4..479ab842b 100644 --- a/crates/streams/src/testing/store.rs +++ b/crates/streams/src/testing/store.rs @@ -198,9 +198,9 @@ async fn a_resend_gets_its_first_positions_and_a_lower_sequence_is_refused() { } } -/// A Producer id is bytes. An id with the field separator and non-ASCII text keeps its own dedupe -/// state, apart from an id that it starts with. -async fn a_producer_id_is_bytes_with_its_own_state() { +/// A Producer id is opaque text. An id with the field separator and non-ASCII text keeps its own +/// dedupe state, apart from an id that it starts with. +async fn a_producer_id_is_text_with_its_own_state() { for target in stores_for(Needs::CALLS).await { target.create().await; let odd = Producer { @@ -254,6 +254,64 @@ async fn a_lower_epoch_is_fenced_for_each_producer_kind() { } } +/// The fence of a Producer is kept by kind and id, and an epoch has no run. So two runs with their +/// own ids keep their own fences, and a lower epoch of the second run is not fenced. A second run +/// that sends the id of the first shares its fence. +async fn two_runs_with_their_own_ids_keep_their_own_fences() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + let first = Run::new("r1").local_activity("1", 20, 1, 1); + let second = Run::new("r2").local_activity("1", 5, 1, 1); + assert_eq!( + (first.id.as_str(), second.id.as_str()), + ("la:r1:1", "la:r2:1") + ); + target.batch(&first, 1, &["r1"]).await.unwrap(); + target.batch(&second, 1, &["r2"]).await.unwrap(); + let reused = Producer { + id: first.id.clone(), + ..second + }; + let stale = target.batch(&reused, 1, &["reused"]).await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + assert_eq!(target.texts().await, ["r1", "r2"]); + } +} + +/// Two Producers of different kinds with one id keep their own fence state. A high epoch of one +/// fences no lower epoch of another, while a lower epoch of the same kind is still fenced. +async fn producers_of_two_kinds_with_one_id_keep_their_own_fences() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + let id = "shared".to_string(); + let local = Producer { + id: id.clone(), + ..Run::new("r1").local_activity("1", 20, 1, 1) + }; + let high = Producer { + id: id.clone(), + ..activity("r1", 5, 50) + }; + let Some(Kind::WorkflowActivity(epoch)) = activity("", 1, 10).kind else { + unreachable!() + }; + let standalone = Producer { + id: id.clone(), + kind: Some(Kind::StandaloneActivity(epoch)), + }; + target.batch(&local, 1, &["local"]).await.unwrap(); + target.batch(&high, 1, &["activity"]).await.unwrap(); + target.batch(&standalone, 1, &["standalone"]).await.unwrap(); + let lower = Producer { + id, + ..activity("r1", 4, 50) + }; + let stale = target.batch(&lower, 1, &["lower"]).await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + assert_eq!(target.texts().await, ["local", "activity", "standalone"]); + } +} + /// A close refuses later appends but not a resend of an earlier batch, and a create of the Stream /// fails as closed. With `STORE_ANSWERS`, a second close returns `already`. async fn a_close_stops_appends_but_not_a_resend() { @@ -718,8 +776,10 @@ pub const CASES: &[super::Case] = &[ case!(closed_and_expired_come_before_already_exists), case!(a_resend_gets_its_first_positions_and_a_lower_sequence_is_refused), case!(a_resend_with_other_content_fails_as_divergent), - case!(a_producer_id_is_bytes_with_its_own_state), + case!(a_producer_id_is_text_with_its_own_state), case!(a_lower_epoch_is_fenced_for_each_producer_kind), + case!(two_runs_with_their_own_ids_keep_their_own_fences), + case!(producers_of_two_kinds_with_one_id_keep_their_own_fences), case!(a_close_stops_appends_but_not_a_resend), case!(a_newer_format_refuses_every_write), case!(an_expired_stream_refuses_writes_and_its_create), diff --git a/crates/streams/src/testing/support.rs b/crates/streams/src/testing/support.rs index 2ae17d5d5..3d4d6573a 100644 --- a/crates/streams/src/testing/support.rs +++ b/crates/streams/src/testing/support.rs @@ -149,14 +149,6 @@ pub(crate) fn record(producer: &Producer, text: &str) -> Vec { .encode_to_vec() } -/// The bytes of a hex text, as the vectors give binary values. -pub(crate) fn hex(text: &str) -> Vec { - (0..text.len()) - .step_by(2) - .map(|at| u8::from_str_radix(&text[at..at + 2], 16).unwrap()) - .collect() -} - /// The digest of the batch that [record] makes of each text. pub(crate) fn digest(texts: &[&str]) -> [u8; crate::digest::DIGEST_LEN] { let records: Vec<_> = texts From 1addba05ba0f6bb6640e23bcb9185a88221c6d10 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Fri, 13 Nov 2026 14:56:22 -0700 Subject: [PATCH 5/5] Wrote the Stream test vectors with a Rust program. The repo has no Python, so the script is now an example of the Stream crate. It is still a second implementation, and it shares no code with the modules that the vector tests check. --- crates/streams/README.md | 7 +- crates/streams/examples/gen_vectors.rs | 494 +++++++++++++++++++++++++ crates/streams/src/ids.rs | 2 +- crates/streams/testdata/gen_vectors.py | 221 ----------- crates/streams/tests/vectors.rs | 2 +- 5 files changed, 500 insertions(+), 226 deletions(-) create mode 100644 crates/streams/examples/gen_vectors.rs delete mode 100644 crates/streams/testdata/gen_vectors.py diff --git a/crates/streams/README.md b/crates/streams/README.md index 9c1bd985d..07f1101e8 100644 --- a/crates/streams/README.md +++ b/crates/streams/README.md @@ -15,9 +15,10 @@ re-exports them, so a value needs no conversion between the crates. `crates/protos/protos/api_upstream`. - `testdata/*.json` are the shared vectors. They cover the Store keys of a Stream id, the meta field of a Producer, offsets, epoch keys and the batch digest of a dedupe entry. - `testdata/gen_vectors.py` writes them from a second implementation, and the crate's tests read - each file. Each Store reads `fields.json` through the shared Store suite. `digests.json` is the - golden of the crate's batch digest. + `examples/gen_vectors.rs` writes them from a second implementation, and the crate's tests read + each file. Run it with `cargo run -p temporalio-streams --example gen_vectors`. It shares no + code with the crate. Each Store reads `fields.json` through the shared Store suite. + `digests.json` is the golden of the crate's batch digest. A Stream id and a Producer id are opaque text. The caller builds both. The runtime refuses an empty id of either kind, and one over 4096 bytes. A Producer id names one Producer in one Stream. diff --git a/crates/streams/examples/gen_vectors.rs b/crates/streams/examples/gen_vectors.rs new file mode 100644 index 000000000..8d525d751 --- /dev/null +++ b/crates/streams/examples/gen_vectors.rs @@ -0,0 +1,494 @@ +//! Writes the shared test vectors of the Stream runtime to `testdata/`. +//! +//! This is a second implementation of the layout. It calls none of the crate's own code for ids, +//! offsets, epochs, fields or digests, so the crate's tests check that code against values it did +//! not compute. The crate's README says which side reads each file. Run it from any directory: +//! +//! ```text +//! cargo run -p temporalio-streams --example gen_vectors +//! ``` + +use sha2::{Digest, Sha256}; +use std::{fs, path::Path}; + +const SEQ_BITS: u32 = 20; + +/// A JSON value that keeps the order of its keys, so the files do not depend on a map type. +enum Json { + Str(String), + Int(i128), + Arr(Vec), + Obj(Vec<(&'static str, Json)>), +} + +impl Json { + fn str(text: impl Into) -> Json { + Json::Str(text.into()) + } + + fn int(value: impl Into) -> Json { + Json::Int(value.into()) + } + + /// Prints with two-space indent and a final newline, one array item to a line. + fn pretty(&self) -> String { + let mut out = String::new(); + self.write(&mut out, 0); + out.push('\n'); + out + } + + fn write(&self, out: &mut String, depth: usize) { + match self { + Json::Str(text) => quote_json(out, text), + Json::Int(value) => out.push_str(&value.to_string()), + Json::Arr(items) if items.is_empty() => out.push_str("[]"), + Json::Obj(fields) if fields.is_empty() => out.push_str("{}"), + Json::Arr(items) => { + out.push_str("[\n"); + for (index, item) in items.iter().enumerate() { + indent(out, depth + 1); + item.write(out, depth + 1); + out.push_str(if index + 1 < items.len() { ",\n" } else { "\n" }); + } + indent(out, depth); + out.push(']'); + } + Json::Obj(fields) => { + out.push_str("{\n"); + for (index, (key, value)) in fields.iter().enumerate() { + indent(out, depth + 1); + quote_json(out, key); + out.push_str(": "); + value.write(out, depth + 1); + out.push_str(if index + 1 < fields.len() { + ",\n" + } else { + "\n" + }); + } + indent(out, depth); + out.push('}'); + } + } + } +} + +fn indent(out: &mut String, depth: usize) { + for _ in 0..depth { + out.push_str(" "); + } +} + +/// Escapes as the readers of the files need, and leaves non-ASCII text as it is. +fn quote_json(out: &mut String, text: &str) { + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{8}' => out.push_str("\\b"), + '\u{c}' => out.push_str("\\f"), + ch if (ch as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", ch as u32)), + ch => out.push(ch), + } + } + out.push('"'); +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Percent-encodes every byte but ASCII letters, digits and `_.-~`. +fn part(text: &str) -> String { + let mut out = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || b"_.-~".contains(&byte) { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +fn ids() -> Json { + let cases = [ + ("temporal-streams", "agent-1:run-1:turn-3"), + ("p:x", "a%3Ab%2Fc:r%251:%C3%A9%20name"), + ("temporal-streams", "{x}:a|b é"), + ]; + // A Stream id is opaque, so the keys take its bytes, whatever they hold. + let streams = cases + .iter() + .map(|(prefix, stream_id)| { + let base = format!("{}:{{{}}}", part(prefix), part(stream_id)); + Json::Obj(vec![ + ("prefix", Json::str(*prefix)), + ("stream_id", Json::str(*stream_id)), + ("meta", Json::str(format!("{base}:meta"))), + ("log", Json::str(format!("{base}:log"))), + ("closed", Json::str(format!("{base}:closed"))), + ]) + }) + .collect(); + Json::Obj(vec![("streams", Json::Arr(streams))]) +} + +fn offsets() -> Json { + let ids = [ + "0-1", + "1-0", + "1700000000000-0", + "1700000000000-7", + "1760000000123-1048575", + ]; + Json::Arr( + ids.iter() + .map(|entry_id| { + let (ms, seq) = split_entry_id(entry_id); + Json::Obj(vec![ + ("entry_id", Json::str(*entry_id)), + ("offset", Json::int((ms << SEQ_BITS) | seq)), + ]) + }) + .collect(), + ) +} + +fn split_entry_id(text: &str) -> (u64, u64) { + let (ms, seq) = text.split_once('-').expect("an entry id has a dash"); + (ms.parse().unwrap(), seq.parse().unwrap()) +} + +/// The fields of an epoch that a case sets. The rest are zero. +#[derive(Default)] +struct Epoch { + task_started_event_id: Option, + task_attempt: Option, + task_started_time: Option<(u64, u32)>, + attempt_scheduled_time: Option<(u64, u32)>, + attempt: Option, +} + +impl Epoch { + fn json(&self) -> Json { + let time = + |(seconds, nanos): (u64, u32)| Json::Arr(vec![Json::int(seconds), Json::int(nanos)]); + let mut fields = Vec::new(); + if let Some(value) = self.task_started_event_id { + fields.push(("task_started_event_id", Json::int(value))); + } + if let Some(value) = self.task_attempt { + fields.push(("task_attempt", Json::int(value))); + } + if let Some(value) = self.task_started_time { + fields.push(("task_started_time", time(value))); + } + if let Some(value) = self.attempt_scheduled_time { + fields.push(("attempt_scheduled_time", time(value))); + } + if let Some(value) = self.attempt { + fields.push(("attempt", Json::int(value))); + } + Json::Obj(fields) + } + + /// 24 bytes, which compare as bytes in the order of the fields. + fn task_key(&self) -> Vec { + let (seconds, nanos) = self.task_started_time.unwrap_or((0, 0)); + let mut key = Vec::with_capacity(24); + key.extend(self.task_started_event_id.unwrap_or(0).to_be_bytes()); + key.extend(self.task_attempt.unwrap_or(0).to_be_bytes()); + key.extend(seconds.to_be_bytes()); + key.extend(nanos.to_be_bytes()); + key + } + + /// 40 bytes, the task key and then the scheduled time and the attempt. + fn key(&self) -> Vec { + let (seconds, nanos) = self.attempt_scheduled_time.unwrap_or((0, 0)); + let mut key = self.task_key(); + key.extend(seconds.to_be_bytes()); + key.extend(nanos.to_be_bytes()); + key.extend(self.attempt.unwrap_or(0).to_be_bytes()); + key + } +} + +fn epochs() -> Json { + let cases = [ + ( + "workflow", + Epoch { + task_started_event_id: Some(8), + task_attempt: Some(2), + task_started_time: Some((1760000000, 5)), + ..Epoch::default() + }, + ), + ( + "local_activity", + Epoch { + task_started_event_id: Some(8), + task_attempt: Some(1), + task_started_time: Some((1760000000, 5)), + attempt: Some(3), + ..Epoch::default() + }, + ), + ( + "activity", + Epoch { + attempt_scheduled_time: Some((1760000001, 999999999)), + attempt: Some(2), + ..Epoch::default() + }, + ), + ("client", Epoch::default()), + ("empty", Epoch::default()), + ]; + Json::Arr( + cases + .iter() + .map(|(kind, epoch)| { + Json::Obj(vec![ + ("kind", Json::str(*kind)), + ("epoch", epoch.json()), + ("key", Json::str(hex(&epoch.key()))), + ("task_key", Json::str(hex(&epoch.task_key()))), + ]) + }) + .collect(), + ) +} + +/// An entry id in a Producer field, the ms in 8 bytes and the seq in 4, zero for none. +fn entry_id(text: &str) -> Vec { + if text.is_empty() { + return vec![0; 12]; + } + let (ms, seq) = split_entry_id(text); + let mut out = ms.to_be_bytes().to_vec(); + out.extend((seq as u32).to_be_bytes()); + out +} + +fn producer_values() -> Json { + let with_records = Epoch { + attempt_scheduled_time: Some((1760000001, 999999999)), + attempt: Some(2), + ..Epoch::default() + }; + let empty = Epoch { + attempt_scheduled_time: Some((1760000002, 0)), + attempt: Some(3), + ..Epoch::default() + }; + let cases = [ + ( + "a batch with Records", + with_records.key(), + 3u64, + "1760000000123-4", + "1760000000123-6", + vec![Record { + data: Some(vec![1]), + ..Record::default() + }], + ), + ( + "an empty batch that wrote no entry", + empty.key(), + 1, + "", + "", + vec![], + ), + ]; + Json::Arr( + cases + .iter() + .map(|(name, fence, sequence, first, last, records)| { + let batch_digest = digest(records); + // The 88 bytes of a `p|` field. + let mut value = fence.clone(); + value.extend(sequence.to_be_bytes()); + value.extend(entry_id(first)); + value.extend(entry_id(last)); + value.extend(&batch_digest); + Json::Obj(vec![ + ("name", Json::str(*name)), + ("fence", Json::str(hex(fence))), + ("sequence", Json::int(*sequence)), + ("first", Json::str(*first)), + ("last", Json::str(*last)), + ("digest", Json::str(hex(&batch_digest))), + ("value", Json::str(hex(&value))), + ]) + }) + .collect(), + ) +} + +fn fields() -> Json { + // A Producer id is opaque, so its field name takes the bytes, also a separator in them. + let producer = "ingest|eu:1 é"; + Json::Obj(vec![ + ("kind", Json::int(3)), + ("producer_id", Json::str(producer)), + ("producer", Json::str(format!("p|3|{producer}"))), + ("producer_values", producer_values()), + ]) +} + +/// A Record of a batch. A key that a case leaves out is `None`, which is not the same as empty. +#[derive(Default)] +struct Record { + metadata: Option)>>, + data: Option>, + external_payloads: Option>, +} + +impl Record { + fn json(&self) -> Json { + let mut fields = Vec::new(); + if let Some(metadata) = &self.metadata { + let entries = metadata + .iter() + .map(|(key, value)| (*key, Json::str(hex(value)))) + .collect(); + fields.push(("metadata", Json::Obj(entries))); + } + if let Some(data) = &self.data { + fields.push(("data", Json::str(hex(data)))); + } + if let Some(sizes) = &self.external_payloads { + let sizes = sizes.iter().map(|size| Json::int(*size)).collect(); + fields.push(("external_payloads", Json::Arr(sizes))); + } + Json::Obj(fields) + } +} + +/// The content digest of a batch, which a dedupe entry keeps, as the first 16 bytes of the +/// SHA-256. +fn digest(records: &[Record]) -> Vec { + let mut out = Sha256::new(); + let part = |out: &mut Sha256, data: &[u8]| { + out.update((data.len() as u64).to_be_bytes()); + out.update(data); + }; + for record in records { + // A Record with none of these keys has no body. + if record.metadata.is_none() && record.data.is_none() && record.external_payloads.is_none() + { + out.update([0u8]); + continue; + } + out.update([1u8]); + let mut metadata: Vec<(&[u8], &[u8])> = record + .metadata + .iter() + .flatten() + .map(|(key, value)| (key.as_bytes(), value.as_slice())) + .collect(); + metadata.sort(); + out.update((metadata.len() as u64).to_be_bytes()); + for (key, value) in metadata { + part(&mut out, key); + part(&mut out, value); + } + part(&mut out, record.data.as_deref().unwrap_or(&[])); + let external = record.external_payloads.as_deref().unwrap_or(&[]); + out.update((external.len() as u64).to_be_bytes()); + for size in external { + out.update(size.to_be_bytes()); + } + } + out.finalize()[..16].to_vec() +} + +fn digests() -> Json { + let data = |bytes: &[u8]| Record { + data: Some(bytes.to_vec()), + ..Record::default() + }; + let cases = [ + ("empty batch", vec![]), + ("one record, no body", vec![Record::default()]), + ("one record, an empty body", vec![data(b"")]), + ("one record", vec![data(b"hello")]), + ( + "metadata in key order", + vec![Record { + metadata: Some(vec![ + ("encoding", b"json/plain".to_vec()), + ("a", vec![0x00, 0xff]), + ]), + data: Some(br#"{"x":1}"#.to_vec()), + ..Record::default() + }], + ), + ("two records", vec![data(&[1]), data(&[2, 3])]), + ( + "external payloads", + vec![Record { + data: Some(vec![1]), + external_payloads: Some(vec![10, 20]), + ..Record::default() + }], + ), + // The length prefixes give these two different digests, though their joined bytes are equal. + ( + "metadata key and value split one way", + vec![Record { + metadata: Some(vec![("ab", b"c".to_vec())]), + ..Record::default() + }], + ), + ( + "metadata key and value split another way", + vec![Record { + metadata: Some(vec![("a", b"bc".to_vec())]), + ..Record::default() + }], + ), + ]; + let sums: Vec> = cases.iter().map(|(_, records)| digest(records)).collect(); + assert_ne!(sums[sums.len() - 1], sums[sums.len() - 2]); + assert_ne!(sums[1], sums[2]); + Json::Arr( + cases + .iter() + .zip(&sums) + .map(|((name, records), sum)| { + Json::Obj(vec![ + ("name", Json::str(*name)), + ( + "records", + Json::Arr(records.iter().map(Record::json).collect()), + ), + ("digest", Json::str(hex(sum))), + ]) + }) + .collect(), + ) +} + +fn main() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata"); + for (name, value) in [ + ("ids.json", ids()), + ("offsets.json", offsets()), + ("epochs.json", epochs()), + ("fields.json", fields()), + ("digests.json", digests()), + ] { + fs::write(dir.join(name), value.pretty()).unwrap(); + } +} diff --git a/crates/streams/src/ids.rs b/crates/streams/src/ids.rs index 432e1e4c6..dc8ca0daa 100644 --- a/crates/streams/src/ids.rs +++ b/crates/streams/src/ids.rs @@ -31,7 +31,7 @@ pub const MAX_PRODUCER_ID_BYTES: usize = 4096; /// The key prefix when the Store options set none. pub const DEFAULT_KEY_PREFIX: &str = "temporal-streams"; -/// Percent-encodes every byte but the unreserved ones, as Python's `quote(text, safe="")` does. +/// Percent-encodes every byte but the unreserved ones, so an id never holds a key separator. pub fn encode_part(text: &str) -> String { let mut out = String::with_capacity(text.len()); for byte in text.bytes() { diff --git a/crates/streams/testdata/gen_vectors.py b/crates/streams/testdata/gen_vectors.py deleted file mode 100644 index a1c030bb1..000000000 --- a/crates/streams/testdata/gen_vectors.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Writes the shared test vectors of the Stream runtime. - -This is a second implementation of the layout, in Python, so the crate's tests check the Rust -code against it. The crate's README says which side reads each file. Run it from this directory: - - python3 gen_vectors.py -""" - -import hashlib -import json -from urllib.parse import quote - -SEQ_BITS = 20 - - -def part(text): - return quote(text, safe="") - - -def keys(prefix, stream_id): - base = f"{part(prefix)}:{{{part(stream_id)}}}" - return { - "meta": f"{base}:meta", - "log": f"{base}:log", - "closed": f"{base}:closed", - } - - -def be64(value): - return max(value, 0).to_bytes(8, "big") - - -def be32(value): - return max(value, 0).to_bytes(4, "big") - - -def task_key(started_event_id=0, attempt=0, started_time=(0, 0)): - """24 bytes, which compare as bytes in the order of the fields.""" - return be64(started_event_id) + be32(attempt) + be64(started_time[0]) + be32(started_time[1]) - - -def epoch_key(epoch): - """40 bytes, the task key and then the scheduled time and the attempt.""" - scheduled = epoch.get("attempt_scheduled_time", (0, 0)) - task = task_key( - epoch.get("task_started_event_id", 0), - epoch.get("task_attempt", 0), - tuple(epoch.get("task_started_time", (0, 0))), - ) - return task + be64(scheduled[0]) + be32(scheduled[1]) + be32(epoch.get("attempt", 0)) - - -def ids(): - """A Stream id is opaque, so the keys take its bytes, whatever they hold.""" - cases = [] - for prefix, stream_id in [ - ("temporal-streams", "agent-1:run-1:turn-3"), - ("p:x", "a%3Ab%2Fc:r%251:%C3%A9%20name"), - ("temporal-streams", "{x}:a|b é"), - ]: - cases.append({"prefix": prefix, "stream_id": stream_id, **keys(prefix, stream_id)}) - return {"streams": cases} - - -def offsets(): - cases = [] - for entry_id in ["0-1", "1-0", "1700000000000-0", "1700000000000-7", "1760000000123-1048575"]: - ms, seq = (int(x) for x in entry_id.split("-")) - cases.append({"entry_id": entry_id, "offset": (ms << SEQ_BITS) | seq}) - return cases - - -def epochs(): - cases = [ - {"kind": "workflow", "epoch": {"task_started_event_id": 8, "task_attempt": 2, - "task_started_time": [1760000000, 5]}}, - {"kind": "local_activity", "epoch": {"task_started_event_id": 8, "task_attempt": 1, - "task_started_time": [1760000000, 5], - "attempt": 3}}, - {"kind": "activity", "epoch": {"attempt_scheduled_time": [1760000001, 999999999], - "attempt": 2}}, - {"kind": "client", "epoch": {}}, - {"kind": "empty", "epoch": {}}, - ] - for case in cases: - case["key"] = epoch_key(case["epoch"]).hex() - epoch = case["epoch"] - case["task_key"] = task_key( - epoch.get("task_started_event_id", 0), - epoch.get("task_attempt", 0), - tuple(epoch.get("task_started_time", (0, 0))), - ).hex() - return cases - - -def entry_id(text): - """An entry id in a Producer field, the ms in 8 bytes and the seq in 4, zero for none.""" - if not text: - return bytes(12) - ms, seq = (int(x) for x in text.split("-")) - return ms.to_bytes(8, "big") + seq.to_bytes(4, "big") - - -def producer_value(fence, sequence, first, last, digest): - """The 88 bytes of a `p|` field.""" - return fence + sequence.to_bytes(8, "big") + entry_id(first) + entry_id(last) + digest - - -def producer_values(): - with_records = epoch_key({"attempt_scheduled_time": [1760000001, 999999999], "attempt": 2}) - empty = epoch_key({"attempt_scheduled_time": [1760000002, 0], "attempt": 3}) - cases = [] - for name, fence, sequence, first, last, records in [ - ("a batch with Records", with_records, 3, "1760000000123-4", "1760000000123-6", - [{"data": "01"}]), - ("an empty batch that wrote no entry", empty, 1, "", "", []), - ]: - batch_digest = bytes.fromhex(digest(records)) - value = producer_value(fence, sequence, first, last, batch_digest) - cases.append({ - "name": name, - "fence": fence.hex(), - "sequence": sequence, - "first": first, - "last": last, - "digest": batch_digest.hex(), - "value": value.hex(), - }) - return cases - - -def fields(): - """A Producer id is opaque, so its field name takes the bytes, also a separator in them.""" - producer = "ingest|eu:1 é" - return { - "kind": 3, - "producer_id": producer, - "producer": f"p|3|{producer}", - "producer_values": producer_values(), - } - - -def digest(records): - """The content digest of a batch, which a dedupe entry keeps, as the first 16 bytes of the - SHA-256.""" - out = hashlib.sha256() - - def part(data): - out.update(len(data).to_bytes(8, "big")) - out.update(data) - - for record in records: - # A record with none of these keys has no body. - if not {"metadata", "data", "external_payloads"} & record.keys(): - out.update(b"\x00") - continue - out.update(b"\x01") - metadata = record.get("metadata", {}) - metadata = sorted((key.encode(), bytes.fromhex(value)) for key, value in metadata.items()) - out.update(len(metadata).to_bytes(8, "big")) - for key, value in metadata: - part(key) - part(value) - part(bytes.fromhex(record.get("data", ""))) - external = record.get("external_payloads", []) - out.update(len(external).to_bytes(8, "big")) - for size in external: - out.update(size.to_bytes(8, "big", signed=True)) - return out.digest()[:16].hex() - - -def digests(): - """Batches as hex bytes, with the hex digest that the crate's `batch_digest` must compute.""" - cases = [ - {"name": "empty batch", "records": []}, - {"name": "one record, no body", "records": [{}]}, - {"name": "one record, an empty body", "records": [{"data": ""}]}, - {"name": "one record", "records": [{"data": b"hello".hex()}]}, - { - "name": "metadata in key order", - "records": [ - { - "metadata": {"encoding": b"json/plain".hex(), "a": "00ff"}, - "data": b'{"x":1}'.hex(), - } - ], - }, - {"name": "two records", "records": [{"data": "01"}, {"data": "0203"}]}, - {"name": "external payloads", "records": [{"data": "01", "external_payloads": [10, 20]}]}, - # The length prefixes give these two different digests, though their joined bytes are equal. - { - "name": "metadata key and value split one way", - "records": [{"metadata": {"ab": "63"}}], - }, - { - "name": "metadata key and value split another way", - "records": [{"metadata": {"a": "6263"}}], - }, - ] - for case in cases: - case["digest"] = digest(case["records"]) - assert cases[-1]["digest"] != cases[-2]["digest"] - assert cases[1]["digest"] != cases[2]["digest"] - return cases - - -def main(): - for name, value in [ - ("ids.json", ids()), - ("offsets.json", offsets()), - ("epochs.json", epochs()), - ("fields.json", fields()), - ("digests.json", digests()), - ]: - with open(name, "w") as out: - json.dump(value, out, indent=2, ensure_ascii=False) - out.write("\n") - - -if __name__ == "__main__": - main() diff --git a/crates/streams/tests/vectors.rs b/crates/streams/tests/vectors.rs index f6b423e59..f1a42ae26 100644 --- a/crates/streams/tests/vectors.rs +++ b/crates/streams/tests/vectors.rs @@ -1,4 +1,4 @@ -//! The shared test vectors, which `testdata/gen_vectors.py` writes from a second implementation. +//! The shared test vectors, which `examples/gen_vectors.rs` writes from a second implementation. //! The crate's README says which other side reads each file. use prost_types::Timestamp;