diff --git a/Cargo.toml b/Cargo.toml index 419d80247..a68c7be8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/macros", "crates/sdk", "crates/workflow", + "crates/streams", "crates/sdk-core-c-bridge", "crates/changelog-release-notes", ] diff --git a/crates/streams/Cargo.toml b/crates/streams/Cargo.toml new file mode 100644 index 000000000..677c94f7e --- /dev/null +++ b/crates/streams/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "temporalio-streams" +version = "0.1.0" +edition = "2024" +rust-version = "1.88.0" +authors = ["Temporal Technologies Inc. "] +license-file = { workspace = true } +description = "The experimental Stream runtime that Temporal SDK bridges link" +homepage = "https://temporal.io/" +repository = "https://github.com/temporalio/sdk-rust" +keywords = ["temporal", "workflow", "streams"] +categories = ["development-tools"] + +[features] +# The shared suites, for the crates of other Stores. +test-support = ["dep:futures-util", "dep:serde_json"] + +[dependencies] +async-trait = "0.1" +futures-util = { version = "0.3", default-features = false, features = ["std"], optional = true } +parking_lot = "0.12" +prost = { workspace = true } +prost-types = { workspace = true } +serde_json = { workspace = true, optional = true } +sha2 = "0.10" +thiserror = { workspace = true } +tokio = { version = "1.47", default-features = false, features = ["rt", "sync", "time"] } +tonic = { workspace = true } + +[dependencies.temporalio-common] +path = "../common" +version = "~1.2.0" +default-features = false + +[dev-dependencies] +futures-util = { version = "0.3", default-features = false, features = ["std"] } +serde_json = { workspace = true } +tokio = { version = "1.47", features = ["macros", "rt-multi-thread", "time"] } +# The integration tests call the memory Store's test hooks, which test-support gates. +temporalio-streams = { path = ".", features = ["test-support"] } + +[lints] +workspace = true diff --git a/crates/streams/README.md b/crates/streams/README.md new file mode 100644 index 000000000..07f1101e8 --- /dev/null +++ b/crates/streams/README.md @@ -0,0 +1,44 @@ +# temporalio-streams + +Experimental. The Stream runtime that the Temporal SDKs link into their bridges. It answers the five +operations of the Stream service on a Store. A bridge builds the runtime per client, as a plain +object, and calls `StreamRuntime::call` with an operation name and the encoded request. The call +gives back the encoded response. Core has no Stream code. + +## Protos and shared vectors + +The crate defines no protos. `temporalio-protos` generates them, and `temporalio_streams::proto` +re-exports them, so a value needs no conversion between the crates. + +- `temporal/api/stream/v1/message.proto`, `temporal/api/streamservice/v1/request_response.proto` + and `temporal/api/enums/v1/stream.proto` are the Stream protos of the API. They are in + `crates/protos/protos/api_upstream`. +- `testdata/*.json` are the shared vectors. They cover the Store keys of a Stream id, the meta + field of a Producer, offsets, epoch keys and the batch digest of a dedupe entry. + `examples/gen_vectors.rs` writes them from a second implementation, and the crate's tests read + each file. Run it with `cargo run -p temporalio-streams --example gen_vectors`. It shares no + code with the crate. Each Store reads `fields.json` through the shared Store suite. + `digests.json` is the golden of the crate's batch digest. + +A Stream id and a Producer id are opaque text. The caller builds both. The runtime refuses an +empty id of either kind, and one over 4096 bytes. A Producer id names one Producer in one Stream. +A Store keeps the fence of a Producer by kind and id, and an epoch has no run. So the id of a +run-based kind must hold its run id. A caller that uses one id in two runs gets `FENCED` in the +second run when that run's epoch is lower. + +## The runtime + +`StreamRuntime` serves the Stream service on one Store, with one typed method for each operation. +`StreamRuntime::call` is the byte form of the same methods. It takes `CreateStream`, +`AppendRecords`, `CloseStream`, `ReadRecords` or `DescribeStream`, and any other name fails with +`Reason::Unsupported`. A Store crate builds the runtime with `StreamRuntime::new`, the name that +lang registered the Store under, and its Store. A `store::LazyStore` connects its inner Store on +the first call that needs it. So a bridge can build the runtime with no async runtime. +`MemoryStreamRuntime::new` builds it on the memory Store, from a `MemoryStreamOptions`. + +## Stores and the shared suites + +The memory Store is for tests. It keeps the key layout of the Redis Store and applies the same rules +in the same order. The `test-support` feature exports the shared suites in `testing`. A Store crate +implements `testing::TestStore` for its Store and runs each suite with `testing::run`, so that +every Store keeps one set of rules. This crate runs them on the memory Store in its unit tests. diff --git a/crates/streams/examples/gen_vectors.rs b/crates/streams/examples/gen_vectors.rs new file mode 100644 index 000000000..8d525d751 --- /dev/null +++ b/crates/streams/examples/gen_vectors.rs @@ -0,0 +1,494 @@ +//! Writes the shared test vectors of the Stream runtime to `testdata/`. +//! +//! This is a second implementation of the layout. It calls none of the crate's own code for ids, +//! offsets, epochs, fields or digests, so the crate's tests check that code against values it did +//! not compute. The crate's README says which side reads each file. Run it from any directory: +//! +//! ```text +//! cargo run -p temporalio-streams --example gen_vectors +//! ``` + +use sha2::{Digest, Sha256}; +use std::{fs, path::Path}; + +const SEQ_BITS: u32 = 20; + +/// A JSON value that keeps the order of its keys, so the files do not depend on a map type. +enum Json { + Str(String), + Int(i128), + Arr(Vec), + Obj(Vec<(&'static str, Json)>), +} + +impl Json { + fn str(text: impl Into) -> Json { + Json::Str(text.into()) + } + + fn int(value: impl Into) -> Json { + Json::Int(value.into()) + } + + /// Prints with two-space indent and a final newline, one array item to a line. + fn pretty(&self) -> String { + let mut out = String::new(); + self.write(&mut out, 0); + out.push('\n'); + out + } + + fn write(&self, out: &mut String, depth: usize) { + match self { + Json::Str(text) => quote_json(out, text), + Json::Int(value) => out.push_str(&value.to_string()), + Json::Arr(items) if items.is_empty() => out.push_str("[]"), + Json::Obj(fields) if fields.is_empty() => out.push_str("{}"), + Json::Arr(items) => { + out.push_str("[\n"); + for (index, item) in items.iter().enumerate() { + indent(out, depth + 1); + item.write(out, depth + 1); + out.push_str(if index + 1 < items.len() { ",\n" } else { "\n" }); + } + indent(out, depth); + out.push(']'); + } + Json::Obj(fields) => { + out.push_str("{\n"); + for (index, (key, value)) in fields.iter().enumerate() { + indent(out, depth + 1); + quote_json(out, key); + out.push_str(": "); + value.write(out, depth + 1); + out.push_str(if index + 1 < fields.len() { + ",\n" + } else { + "\n" + }); + } + indent(out, depth); + out.push('}'); + } + } + } +} + +fn indent(out: &mut String, depth: usize) { + for _ in 0..depth { + out.push_str(" "); + } +} + +/// Escapes as the readers of the files need, and leaves non-ASCII text as it is. +fn quote_json(out: &mut String, text: &str) { + out.push('"'); + for ch in text.chars() { + match ch { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{8}' => out.push_str("\\b"), + '\u{c}' => out.push_str("\\f"), + ch if (ch as u32) < 0x20 => out.push_str(&format!("\\u{:04x}", ch as u32)), + ch => out.push(ch), + } + } + out.push('"'); +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// Percent-encodes every byte but ASCII letters, digits and `_.-~`. +fn part(text: &str) -> String { + let mut out = String::new(); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || b"_.-~".contains(&byte) { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +fn ids() -> Json { + let cases = [ + ("temporal-streams", "agent-1:run-1:turn-3"), + ("p:x", "a%3Ab%2Fc:r%251:%C3%A9%20name"), + ("temporal-streams", "{x}:a|b é"), + ]; + // A Stream id is opaque, so the keys take its bytes, whatever they hold. + let streams = cases + .iter() + .map(|(prefix, stream_id)| { + let base = format!("{}:{{{}}}", part(prefix), part(stream_id)); + Json::Obj(vec![ + ("prefix", Json::str(*prefix)), + ("stream_id", Json::str(*stream_id)), + ("meta", Json::str(format!("{base}:meta"))), + ("log", Json::str(format!("{base}:log"))), + ("closed", Json::str(format!("{base}:closed"))), + ]) + }) + .collect(); + Json::Obj(vec![("streams", Json::Arr(streams))]) +} + +fn offsets() -> Json { + let ids = [ + "0-1", + "1-0", + "1700000000000-0", + "1700000000000-7", + "1760000000123-1048575", + ]; + Json::Arr( + ids.iter() + .map(|entry_id| { + let (ms, seq) = split_entry_id(entry_id); + Json::Obj(vec![ + ("entry_id", Json::str(*entry_id)), + ("offset", Json::int((ms << SEQ_BITS) | seq)), + ]) + }) + .collect(), + ) +} + +fn split_entry_id(text: &str) -> (u64, u64) { + let (ms, seq) = text.split_once('-').expect("an entry id has a dash"); + (ms.parse().unwrap(), seq.parse().unwrap()) +} + +/// The fields of an epoch that a case sets. The rest are zero. +#[derive(Default)] +struct Epoch { + task_started_event_id: Option, + task_attempt: Option, + task_started_time: Option<(u64, u32)>, + attempt_scheduled_time: Option<(u64, u32)>, + attempt: Option, +} + +impl Epoch { + fn json(&self) -> Json { + let time = + |(seconds, nanos): (u64, u32)| Json::Arr(vec![Json::int(seconds), Json::int(nanos)]); + let mut fields = Vec::new(); + if let Some(value) = self.task_started_event_id { + fields.push(("task_started_event_id", Json::int(value))); + } + if let Some(value) = self.task_attempt { + fields.push(("task_attempt", Json::int(value))); + } + if let Some(value) = self.task_started_time { + fields.push(("task_started_time", time(value))); + } + if let Some(value) = self.attempt_scheduled_time { + fields.push(("attempt_scheduled_time", time(value))); + } + if let Some(value) = self.attempt { + fields.push(("attempt", Json::int(value))); + } + Json::Obj(fields) + } + + /// 24 bytes, which compare as bytes in the order of the fields. + fn task_key(&self) -> Vec { + let (seconds, nanos) = self.task_started_time.unwrap_or((0, 0)); + let mut key = Vec::with_capacity(24); + key.extend(self.task_started_event_id.unwrap_or(0).to_be_bytes()); + key.extend(self.task_attempt.unwrap_or(0).to_be_bytes()); + key.extend(seconds.to_be_bytes()); + key.extend(nanos.to_be_bytes()); + key + } + + /// 40 bytes, the task key and then the scheduled time and the attempt. + fn key(&self) -> Vec { + let (seconds, nanos) = self.attempt_scheduled_time.unwrap_or((0, 0)); + let mut key = self.task_key(); + key.extend(seconds.to_be_bytes()); + key.extend(nanos.to_be_bytes()); + key.extend(self.attempt.unwrap_or(0).to_be_bytes()); + key + } +} + +fn epochs() -> Json { + let cases = [ + ( + "workflow", + Epoch { + task_started_event_id: Some(8), + task_attempt: Some(2), + task_started_time: Some((1760000000, 5)), + ..Epoch::default() + }, + ), + ( + "local_activity", + Epoch { + task_started_event_id: Some(8), + task_attempt: Some(1), + task_started_time: Some((1760000000, 5)), + attempt: Some(3), + ..Epoch::default() + }, + ), + ( + "activity", + Epoch { + attempt_scheduled_time: Some((1760000001, 999999999)), + attempt: Some(2), + ..Epoch::default() + }, + ), + ("client", Epoch::default()), + ("empty", Epoch::default()), + ]; + Json::Arr( + cases + .iter() + .map(|(kind, epoch)| { + Json::Obj(vec![ + ("kind", Json::str(*kind)), + ("epoch", epoch.json()), + ("key", Json::str(hex(&epoch.key()))), + ("task_key", Json::str(hex(&epoch.task_key()))), + ]) + }) + .collect(), + ) +} + +/// An entry id in a Producer field, the ms in 8 bytes and the seq in 4, zero for none. +fn entry_id(text: &str) -> Vec { + if text.is_empty() { + return vec![0; 12]; + } + let (ms, seq) = split_entry_id(text); + let mut out = ms.to_be_bytes().to_vec(); + out.extend((seq as u32).to_be_bytes()); + out +} + +fn producer_values() -> Json { + let with_records = Epoch { + attempt_scheduled_time: Some((1760000001, 999999999)), + attempt: Some(2), + ..Epoch::default() + }; + let empty = Epoch { + attempt_scheduled_time: Some((1760000002, 0)), + attempt: Some(3), + ..Epoch::default() + }; + let cases = [ + ( + "a batch with Records", + with_records.key(), + 3u64, + "1760000000123-4", + "1760000000123-6", + vec![Record { + data: Some(vec![1]), + ..Record::default() + }], + ), + ( + "an empty batch that wrote no entry", + empty.key(), + 1, + "", + "", + vec![], + ), + ]; + Json::Arr( + cases + .iter() + .map(|(name, fence, sequence, first, last, records)| { + let batch_digest = digest(records); + // The 88 bytes of a `p|` field. + let mut value = fence.clone(); + value.extend(sequence.to_be_bytes()); + value.extend(entry_id(first)); + value.extend(entry_id(last)); + value.extend(&batch_digest); + Json::Obj(vec![ + ("name", Json::str(*name)), + ("fence", Json::str(hex(fence))), + ("sequence", Json::int(*sequence)), + ("first", Json::str(*first)), + ("last", Json::str(*last)), + ("digest", Json::str(hex(&batch_digest))), + ("value", Json::str(hex(&value))), + ]) + }) + .collect(), + ) +} + +fn fields() -> Json { + // A Producer id is opaque, so its field name takes the bytes, also a separator in them. + let producer = "ingest|eu:1 é"; + Json::Obj(vec![ + ("kind", Json::int(3)), + ("producer_id", Json::str(producer)), + ("producer", Json::str(format!("p|3|{producer}"))), + ("producer_values", producer_values()), + ]) +} + +/// A Record of a batch. A key that a case leaves out is `None`, which is not the same as empty. +#[derive(Default)] +struct Record { + metadata: Option)>>, + data: Option>, + external_payloads: Option>, +} + +impl Record { + fn json(&self) -> Json { + let mut fields = Vec::new(); + if let Some(metadata) = &self.metadata { + let entries = metadata + .iter() + .map(|(key, value)| (*key, Json::str(hex(value)))) + .collect(); + fields.push(("metadata", Json::Obj(entries))); + } + if let Some(data) = &self.data { + fields.push(("data", Json::str(hex(data)))); + } + if let Some(sizes) = &self.external_payloads { + let sizes = sizes.iter().map(|size| Json::int(*size)).collect(); + fields.push(("external_payloads", Json::Arr(sizes))); + } + Json::Obj(fields) + } +} + +/// The content digest of a batch, which a dedupe entry keeps, as the first 16 bytes of the +/// SHA-256. +fn digest(records: &[Record]) -> Vec { + let mut out = Sha256::new(); + let part = |out: &mut Sha256, data: &[u8]| { + out.update((data.len() as u64).to_be_bytes()); + out.update(data); + }; + for record in records { + // A Record with none of these keys has no body. + if record.metadata.is_none() && record.data.is_none() && record.external_payloads.is_none() + { + out.update([0u8]); + continue; + } + out.update([1u8]); + let mut metadata: Vec<(&[u8], &[u8])> = record + .metadata + .iter() + .flatten() + .map(|(key, value)| (key.as_bytes(), value.as_slice())) + .collect(); + metadata.sort(); + out.update((metadata.len() as u64).to_be_bytes()); + for (key, value) in metadata { + part(&mut out, key); + part(&mut out, value); + } + part(&mut out, record.data.as_deref().unwrap_or(&[])); + let external = record.external_payloads.as_deref().unwrap_or(&[]); + out.update((external.len() as u64).to_be_bytes()); + for size in external { + out.update(size.to_be_bytes()); + } + } + out.finalize()[..16].to_vec() +} + +fn digests() -> Json { + let data = |bytes: &[u8]| Record { + data: Some(bytes.to_vec()), + ..Record::default() + }; + let cases = [ + ("empty batch", vec![]), + ("one record, no body", vec![Record::default()]), + ("one record, an empty body", vec![data(b"")]), + ("one record", vec![data(b"hello")]), + ( + "metadata in key order", + vec![Record { + metadata: Some(vec![ + ("encoding", b"json/plain".to_vec()), + ("a", vec![0x00, 0xff]), + ]), + data: Some(br#"{"x":1}"#.to_vec()), + ..Record::default() + }], + ), + ("two records", vec![data(&[1]), data(&[2, 3])]), + ( + "external payloads", + vec![Record { + data: Some(vec![1]), + external_payloads: Some(vec![10, 20]), + ..Record::default() + }], + ), + // The length prefixes give these two different digests, though their joined bytes are equal. + ( + "metadata key and value split one way", + vec![Record { + metadata: Some(vec![("ab", b"c".to_vec())]), + ..Record::default() + }], + ), + ( + "metadata key and value split another way", + vec![Record { + metadata: Some(vec![("a", b"bc".to_vec())]), + ..Record::default() + }], + ), + ]; + let sums: Vec> = cases.iter().map(|(_, records)| digest(records)).collect(); + assert_ne!(sums[sums.len() - 1], sums[sums.len() - 2]); + assert_ne!(sums[1], sums[2]); + Json::Arr( + cases + .iter() + .zip(&sums) + .map(|((name, records), sum)| { + Json::Obj(vec![ + ("name", Json::str(*name)), + ( + "records", + Json::Arr(records.iter().map(Record::json).collect()), + ), + ("digest", Json::str(hex(sum))), + ]) + }) + .collect(), + ) +} + +fn main() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("testdata"); + for (name, value) in [ + ("ids.json", ids()), + ("offsets.json", offsets()), + ("epochs.json", epochs()), + ("fields.json", fields()), + ("digests.json", digests()), + ] { + fs::write(dir.join(name), value.pretty()).unwrap(); + } +} diff --git a/crates/streams/src/digest.rs b/crates/streams/src/digest.rs new file mode 100644 index 000000000..ef448b357 --- /dev/null +++ b/crates/streams/src/digest.rs @@ -0,0 +1,44 @@ +//! The content digest of a batch. Each dedupe entry stores it, so that a resend with the same +//! `sequence` and other content fails instead of being dropped without an error. The Store compares +//! the digest that this function gives, and `testdata/digests.json` has its expected values. + +use crate::proto::AppendRecord; +use sha2::{Digest, Sha256}; + +/// The length of a [batch_digest]. +pub const DIGEST_LEN: usize = 16; + +/// The first 16 bytes of a SHA-256 over the Records in order. Each part has its length before it, +/// so that two different batches never hash the same bytes. The metadata is hashed in key order, +/// since a map has no order on the wire. +pub fn batch_digest(records: &[AppendRecord]) -> [u8; DIGEST_LEN] { + let mut hasher = Sha256::new(); + for record in records { + // A Record with no body and one with an empty body are two different stored Records. + let Some(body) = &record.body else { + hasher.update([0]); + continue; + }; + hasher.update([1]); + let mut metadata: Vec<_> = body.metadata.iter().collect(); + metadata.sort(); + hasher.update((metadata.len() as u64).to_be_bytes()); + for (key, value) in metadata { + part(&mut hasher, key.as_bytes()); + part(&mut hasher, value); + } + part(&mut hasher, &body.data); + hasher.update((body.external_payloads.len() as u64).to_be_bytes()); + for external in &body.external_payloads { + hasher.update(external.size_bytes.to_be_bytes()); + } + } + let mut digest = [0; DIGEST_LEN]; + digest.copy_from_slice(&hasher.finalize()[..DIGEST_LEN]); + digest +} + +fn part(hasher: &mut Sha256, bytes: &[u8]) { + hasher.update((bytes.len() as u64).to_be_bytes()); + hasher.update(bytes); +} diff --git a/crates/streams/src/dispatch.rs b/crates/streams/src/dispatch.rs new file mode 100644 index 000000000..6acec56f5 --- /dev/null +++ b/crates/streams/src/dispatch.rs @@ -0,0 +1,172 @@ +//! The byte form of the runtime's calls, for bridges. A bridge passes the encoded request and gets +//! the encoded response, so it decodes no proto. + +use crate::{Reason, StreamError, StreamResult, StreamRuntime}; +use prost::Message; + +fn decode(input: &[u8]) -> StreamResult { + M::decode(input).map_err(|error| StreamError::invalid(format!("The request is bad: {error}"))) +} + +impl StreamRuntime { + /// Runs one of the five Stream service operations on an encoded request, by its name, and + /// returns the encoded response. The names are `CreateStream`, `AppendRecords`, `CloseStream`, + /// `ReadRecords` and `DescribeStream`. The bridge raises an error with [StreamError::code] and + /// [StreamError::details]. + pub async fn call(&self, operation: &str, request: &[u8]) -> StreamResult> { + let response = match operation { + "CreateStream" => self.create_stream(decode(request)?).await?.encode_to_vec(), + "AppendRecords" => self.append_records(decode(request)?).await?.encode_to_vec(), + "CloseStream" => self.close_stream(decode(request)?).await?.encode_to_vec(), + "ReadRecords" => self.read_records(decode(request)?).await?.encode_to_vec(), + "DescribeStream" => self + .describe_stream(decode(request)?) + .await? + .encode_to_vec(), + _ => { + return Err(StreamError::new( + Reason::Unsupported, + format!("Unknown operation {operation}"), + )); + } + }; + Ok(response) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + MemoryStreamOptions, MemoryStreamRuntime, + proto::{ + AppendRecord, AppendRecordsRequest, AppendRecordsResponse, ClientProducer, + CloseStreamRequest, CloseStreamResponse, CreateStreamRequest, CreateStreamResponse, + DescribeStreamRequest, DescribeStreamResponse, Producer, ReadRecordsRequest, + ReadRecordsResponse, StreamRef, StreamStatus, producer::Kind, + }, + }; + use std::sync::Arc; + use temporalio_common::protos::temporal::api::common::v1::Payload; + use tonic::Code; + + const OPERATIONS: [&str; 5] = [ + "CreateStream", + "AppendRecords", + "CloseStream", + "ReadRecords", + "DescribeStream", + ]; + + fn runtime() -> Arc { + MemoryStreamRuntime::new("b", MemoryStreamOptions::default(), None) + } + + fn stream_ref() -> Option { + Some(StreamRef { + id: "s".to_string(), + ..Default::default() + }) + } + + fn client() -> Option { + Some(Producer { + id: "client:me:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + }) + } + + fn records() -> Vec { + vec![AppendRecord { + body: Some(Payload::default()), + }] + } + + async fn call( + runtime: &StreamRuntime, + operation: &str, + request: Req, + ) -> Resp { + let bytes = runtime + .call(operation, &request.encode_to_vec()) + .await + .unwrap(); + Resp::decode(bytes.as_slice()).unwrap() + } + + async fn create(runtime: &StreamRuntime) -> CreateStreamResponse { + let request = CreateStreamRequest { + stream_ref: stream_ref(), + ..Default::default() + }; + call(runtime, "CreateStream", request).await + } + + // A typo in an operation name would make it unknown, which is unsupported, so the loop also + // checks the names that a bridge calls. + #[tokio::test] + async fn bad_bytes_are_invalid_and_an_unknown_operation_is_unsupported() { + let runtime = runtime(); + for operation in OPERATIONS { + let error = runtime.call(operation, &[0xff]).await.unwrap_err(); + assert_eq!(error.reason, Reason::Invalid, "{operation}"); + } + let error = runtime.call("Nope", &[]).await.unwrap_err(); + assert_eq!( + (error.reason, error.code(), error.message.as_str()), + ( + Reason::Unsupported, + Code::Unimplemented, + "Unknown operation Nope" + ) + ); + } + + #[tokio::test] + async fn each_stream_operation_round_trips_its_bytes() { + let runtime = runtime(); + assert_eq!(create(&runtime).await.r#ref.unwrap().store, "b"); + let appended: AppendRecordsResponse = call( + &runtime, + "AppendRecords", + AppendRecordsRequest { + r#ref: stream_ref(), + producer: client(), + sequence: 1, + records: records(), + ..Default::default() + }, + ) + .await; + let read: ReadRecordsResponse = call( + &runtime, + "ReadRecords", + ReadRecordsRequest { + r#ref: stream_ref(), + ..Default::default() + }, + ) + .await; + assert_eq!(read.last_offset, appended.last_offset); + let _: CloseStreamResponse = call( + &runtime, + "CloseStream", + CloseStreamRequest { + r#ref: stream_ref(), + producer: client(), + ..Default::default() + }, + ) + .await; + let described: DescribeStreamResponse = call( + &runtime, + "DescribeStream", + DescribeStreamRequest { + r#ref: stream_ref(), + ..Default::default() + }, + ) + .await; + assert_eq!(described.status, StreamStatus::Closed as i32); + } +} diff --git a/crates/streams/src/epoch.rs b/crates/streams/src/epoch.rs new file mode 100644 index 000000000..d8babde68 --- /dev/null +++ b/crates/streams/src/epoch.rs @@ -0,0 +1,139 @@ +//! Producer epochs and their order. +//! +//! A Store keeps an epoch as a key of fixed-width big-endian fields, in the order of comparison. +//! A byte comparison of two keys then orders the epochs field by field. `testdata/epochs.json` has +//! the shared vectors of the keys, in hex. + +use crate::{ + StreamError, StreamResult, + proto::{Producer, WorkflowTaskEpoch, producer::Kind}, +}; +use prost_types::Timestamp; + +/// The kind of a Producer, which is the case of its epoch. The stored field names and keys hold +/// its number, so each kind keeps its number. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ProducerKind { + /// A Workflow run. + Workflow = 1, + /// A Local Activity. + LocalActivity = 2, + /// An Activity of a Workflow. + Activity = 3, + /// A client. + Client = 4, + /// A Standalone Activity. + StandaloneActivity = 5, +} + +impl From<&Kind> for ProducerKind { + fn from(kind: &Kind) -> Self { + match kind { + Kind::Workflow(_) => Self::Workflow, + Kind::LocalActivity(_) => Self::LocalActivity, + Kind::WorkflowActivity(_) => Self::Activity, + Kind::StandaloneActivity(_) => Self::StandaloneActivity, + Kind::Client(_) => Self::Client, + } + } +} + +/// Refuses a Producer with no kind case. +impl TryFrom<&Producer> for ProducerKind { + type Error = StreamError; + + fn try_from(producer: &Producer) -> StreamResult { + kind_of(producer).ok_or_else(|| StreamError::invalid("The Producer has no kind")) + } +} + +/// The kind case of a Producer, `None` when it has none. +pub fn kind_of(producer: &Producer) -> Option { + producer.kind.as_ref().map(ProducerKind::from) +} + +/// The length of a [task_key]. +pub const TASK_KEY_LEN: usize = 24; +/// The length of an [epoch_key]. +pub const EPOCH_KEY_LEN: usize = 40; + +// A negative field is clamped at zero, since its bytes would outrank every real epoch. +fn be64(value: i64) -> [u8; 8] { + value.max(0).to_be_bytes() +} + +fn be32(value: i32) -> [u8; 4] { + value.max(0).to_be_bytes() +} + +fn time_fields(time: Option<&Timestamp>) -> (i64, i32) { + time.map_or((0, 0), |time| (time.seconds, time.nanos)) +} + +/// The key of a Workflow Task attempt. It is the started event id, the attempt, and the started +/// seconds and nanos, in 8, 4, 8 and 4 bytes. The epoch keys of the run and of the task's Local +/// Activities start with it. +pub fn task_key(task: &WorkflowTaskEpoch) -> [u8; TASK_KEY_LEN] { + let (seconds, nanos) = time_fields(task.started_time.as_ref()); + let mut key = [0; TASK_KEY_LEN]; + key[..8].copy_from_slice(&be64(task.started_event_id)); + key[8..12].copy_from_slice(&be32(task.attempt)); + key[12..20].copy_from_slice(&be64(seconds)); + key[20..].copy_from_slice(&be32(nanos)); + key +} + +/// The key of an epoch. It is the [task_of] key, then the attempt's scheduled seconds and nanos +/// and the attempt, in 8, 4 and 4 bytes. A field that the case doesn't have is zero, and a Producer +/// with no case is all zeros. +pub fn epoch_key(kind: Option<&Kind>) -> [u8; EPOCH_KEY_LEN] { + let (scheduled, attempt) = match kind { + Some(Kind::LocalActivity(local)) => (None, local.epoch.map_or(0, |e| e.attempt)), + Some(Kind::WorkflowActivity(activity) | Kind::StandaloneActivity(activity)) => { + let epoch = activity.epoch; + ( + epoch.and_then(|e| e.attempt_scheduled_time), + epoch.map_or(0, |e| e.attempt), + ) + } + // A client has one epoch for its whole life, so nothing fences it. + Some(Kind::Client(_)) => (None, 0), + Some(Kind::Workflow(_)) | None => (None, 0), + }; + let (seconds, nanos) = time_fields(scheduled.as_ref()); + let mut key = [0; EPOCH_KEY_LEN]; + key[..TASK_KEY_LEN].copy_from_slice(&task_key(&task_of(kind))); + key[24..32].copy_from_slice(&be64(seconds)); + key[32..36].copy_from_slice(&be32(nanos)); + key[36..].copy_from_slice(&be32(attempt)); + key +} + +/// The Workflow Task of an epoch. It is a Workflow's own task, or the task that dispatched a Local +/// Activity, and zero for the other kinds. +pub fn task_of(kind: Option<&Kind>) -> WorkflowTaskEpoch { + match kind { + Some(Kind::Workflow(w)) => w.epoch.unwrap_or_default(), + Some(Kind::LocalActivity(local)) => local.epoch.and_then(|e| e.task).unwrap_or_default(), + _ => WorkflowTaskEpoch::default(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::proto::{LocalActivityEpoch, LocalActivityProducer}; + + // Without the clamp, the bytes of a negative field would outrank every real epoch. + #[test] + fn a_negative_field_keys_as_zero() { + let negative = Kind::LocalActivity(LocalActivityProducer { + epoch: Some(LocalActivityEpoch { + task: None, + attempt: -1, + }), + ..Default::default() + }); + assert_eq!(epoch_key(Some(&negative)), epoch_key(None)); + } +} diff --git a/crates/streams/src/error.rs b/crates/streams/src/error.rs new file mode 100644 index 000000000..527015956 --- /dev/null +++ b/crates/streams/src/error.rs @@ -0,0 +1,172 @@ +//! The error of every failed call, and its form for lang. + +use crate::proto::{StreamErrorDetails, StreamErrorReason}; +use prost::Message; +use tonic::Code; + +/// The result of every fallible call in the crate. +pub type StreamResult = Result; + +/// Why a call failed. Each reason has one status code and one case of `StreamErrorReason`, and lang +/// maps the reason to one error class. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Reason { + /// Nobody created the Stream. + NotFound, + /// The Stream's Retention or last-write TTL passed. + Expired, + /// The Stream is closed. + Closed, + /// The Producer's epoch is lower than the newest one. + Fenced, + /// The batch has a lower `sequence` than the last one of its epoch. + SequenceTooLow, + /// The batch has the `sequence` of the last one of its epoch, and other content. + SeqDivergent, + /// The Stream's owner doesn't allow this Producer to close it. + NotOwner, + /// A newer format version wrote the Stream. + FormatTooNew, + /// The offset is past the newest offset, for example after a failover lost the newest Records. + OffsetTooHigh, + /// A Record or a batch is over the Store's limit. + TooLarge, + /// The request is malformed. + Invalid, + /// The Store failed or did not answer. A retry can succeed. + StorageUnavailable, + /// The Store lacks a capability that the call needs. + Unsupported, + /// A create that doesn't use an existing Stream names a Stream that exists. + AlreadyExists, + /// The Store gave back data that isn't valid. A retry reads the same data. + StorageCorrupt, +} + +impl Reason { + /// The token that starts the error's text. Lang reads the reason from + /// `StreamErrorDetails.reason`, not from this text. + pub fn token(self) -> &'static str { + match self { + Reason::NotFound => "stream_not_found", + Reason::Expired => "stream_expired", + Reason::Closed => "stream_closed", + Reason::Fenced => "stream_fenced", + Reason::SequenceTooLow => "stream_sequence_too_low", + Reason::SeqDivergent => "seq_divergent", + Reason::NotOwner => "stream_not_owner", + Reason::FormatTooNew => "stream_format_too_new", + Reason::OffsetTooHigh => "stream_offset_too_high", + Reason::TooLarge => "stream_too_large", + Reason::Invalid => "stream_invalid", + Reason::StorageUnavailable => "stream_storage_unavailable", + Reason::Unsupported => "stream_unsupported", + Reason::AlreadyExists => "stream_already_exists", + Reason::StorageCorrupt => "stream_storage_corrupt", + } + } + + /// The reason in the error details. + pub fn proto(self) -> StreamErrorReason { + match self { + Reason::NotFound => StreamErrorReason::NotFound, + Reason::Expired => StreamErrorReason::Expired, + Reason::Closed => StreamErrorReason::Closed, + Reason::Fenced => StreamErrorReason::Fenced, + Reason::SequenceTooLow => StreamErrorReason::SequenceTooLow, + Reason::SeqDivergent => StreamErrorReason::SeqDivergent, + Reason::NotOwner => StreamErrorReason::NotOwner, + Reason::FormatTooNew => StreamErrorReason::FormatTooNew, + Reason::OffsetTooHigh => StreamErrorReason::OffsetTooHigh, + Reason::TooLarge => StreamErrorReason::TooLarge, + Reason::Invalid => StreamErrorReason::Invalid, + Reason::StorageUnavailable => StreamErrorReason::StorageUnavailable, + Reason::Unsupported => StreamErrorReason::Unsupported, + Reason::AlreadyExists => StreamErrorReason::AlreadyExists, + Reason::StorageCorrupt => StreamErrorReason::StorageCorrupt, + } + } + + /// The status code of the reason, as in the proposal's Appendix A. + pub fn code(self) -> Code { + match self { + Reason::NotFound | Reason::Expired => Code::NotFound, + Reason::Closed + | Reason::Fenced + | Reason::SequenceTooLow + | Reason::SeqDivergent + | Reason::NotOwner + | Reason::FormatTooNew => Code::FailedPrecondition, + Reason::TooLarge | Reason::Invalid => Code::InvalidArgument, + // A read from the offset works again once the Stream has Records up to it. + Reason::OffsetTooHigh => Code::OutOfRange, + Reason::StorageUnavailable => Code::Unavailable, + Reason::Unsupported => Code::Unimplemented, + Reason::AlreadyExists => Code::AlreadyExists, + Reason::StorageCorrupt => Code::DataLoss, + } + } +} + +/// A failed Stream call. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +#[error("{}: {message}", reason.token())] +pub struct StreamError { + /// Why the call failed. + pub reason: Reason, + /// What happened, for a log or the facade error. + pub message: String, +} + +impl StreamError { + /// Creates an error. + pub fn new(reason: Reason, message: impl Into) -> Self { + Self { + reason, + message: message.into(), + } + } + + /// A malformed request. + pub fn invalid(message: impl Into) -> Self { + Self::new(Reason::Invalid, message) + } + + /// A Store that failed or did not answer. + pub fn storage_unavailable(message: impl Into) -> Self { + Self::new(Reason::StorageUnavailable, message) + } + + /// Store data that isn't valid. + pub fn storage_corrupt(message: impl Into) -> Self { + Self::new(Reason::StorageCorrupt, message) + } + + /// The status code that lang raises the error with. + pub fn code(&self) -> Code { + self.reason.code() + } + + /// The encoded [StreamErrorDetails], which lang reads the reason from, as from the details of + /// a gRPC status. + pub fn details(&self) -> Vec { + StreamErrorDetails { + reason: self.reason.proto() as i32, + } + .encode_to_vec() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_error_carries_its_code_and_reason_to_lang() { + let error = StreamError::new(Reason::FormatTooNew, "format 2 is newer than 1"); + assert_eq!(error.code(), Code::FailedPrecondition); + let details = StreamErrorDetails::decode(error.details().as_slice()).unwrap(); + assert_eq!(details.reason(), StreamErrorReason::FormatTooNew); + } +} diff --git a/crates/streams/src/ids.rs b/crates/streams/src/ids.rs new file mode 100644 index 000000000..dc8ca0daa --- /dev/null +++ b/crates/streams/src/ids.rs @@ -0,0 +1,447 @@ +//! The checks of Stream ids and Producers, and the Redis key layout. A caller builds both kinds of +//! id, so the crate treats each as opaque text. `testdata/ids.json` has the shared vectors of the +//! key layout. +//! +//! ```text +//! :{}:meta the create key, the fence, dedupe and close state +//! :{}:log the log, one entry per Record +//! :{}:closed the entry that a close adds, which wakes a waiting read +//! ``` +//! +//! A key has no Temporal namespace, so a Stream id is unique in one Store and key prefix. The part +//! in braces is the Redis Cluster hash tag, so that the keys are in one slot for one script. Each +//! part is percent-encoded, so a `:` or a brace in an id can't make two Streams share a key. + +use crate::{ + StreamError, StreamResult, + epoch::ProducerKind, + proto::{Producer, WorkflowTaskEpoch, producer::Kind}, +}; + +/// The longest Stream id, in bytes. +pub const MAX_STREAM_ID_BYTES: usize = 4096; + +/// The longest Producer id, in bytes. A caller builds the id from names that Temporal limits to +/// 1000 characters, so an id with two of them still fits. A Store keeps the id in one meta field of +/// each Producer for as long as the Stream lives. Each Record of a batch holds its Producer, so the +/// size check counts the Producer once for each Record against +/// [MAX_BATCH_BYTES](crate::store::MAX_BATCH_BYTES). +pub const MAX_PRODUCER_ID_BYTES: usize = 4096; + +/// The key prefix when the Store options set none. +pub const DEFAULT_KEY_PREFIX: &str = "temporal-streams"; + +/// Percent-encodes every byte but the unreserved ones, so an id never holds a key separator. +pub fn encode_part(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~') { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +/// Refuses an empty or too long Stream id. +pub fn check_stream_id(id: &str) -> StreamResult<()> { + if id.is_empty() { + return Err(StreamError::invalid("The Stream id is empty")); + } + if id.len() > MAX_STREAM_ID_BYTES { + return Err(StreamError::invalid(format!( + "The Stream id has {} bytes, more than {MAX_STREAM_ID_BYTES}", + id.len() + ))); + } + Ok(()) +} + +/// Refuses a Producer with no id, an id over [MAX_PRODUCER_ID_BYTES], or no kind. The id is opaque +/// text, so the Store requires no shape of it. It names one Producer in one Stream. A Store keeps +/// the fence of a Producer by kind and id, and an epoch has no run. So the id of a run-based kind +/// must hold its run id. A caller that uses one id in two runs gets `FENCED` in the second run when +/// that run's epoch is lower. +pub fn check_producer(producer: &Producer) -> StreamResult<()> { + if producer.id.is_empty() { + return Err(StreamError::invalid("The Producer id is empty")); + } + if producer.id.len() > MAX_PRODUCER_ID_BYTES { + return Err(StreamError::invalid(format!( + "The Producer id has {} bytes, more than {MAX_PRODUCER_ID_BYTES}", + producer.id.len() + ))); + } + ProducerKind::try_from(producer)?; + Ok(()) +} + +fn missing(what: &str) -> StreamResult<()> { + Err(StreamError::invalid(format!("The Producer has no {what}"))) +} + +fn at_least_one(attempt: i32, of: &str) -> StreamResult<()> { + if attempt < 1 { + return Err(StreamError::invalid(format!( + "The {of} attempt is {attempt}, and it must be 1 or more" + ))); + } + Ok(()) +} + +/// Refuses a Workflow Task epoch with no started event id, or with an attempt below 1. +fn check_task(task: &WorkflowTaskEpoch) -> StreamResult<()> { + // Without its started event, a later task would have the epoch of an earlier one. + if task.started_event_id <= 0 { + return Err(StreamError::invalid( + "The Workflow Task of the epoch has no started event id", + )); + } + at_least_one(task.attempt, "Workflow Task") +} + +/// Refuses a Producer that lacks a field the API requires. A Workflow Producer only closes a +/// Stream, so it may send no epoch, but an epoch that it sends must order. A Workflow Task's +/// started time stays optional, since Core leaves it empty when the server gave the task none. +pub fn check_required(producer: &Producer) -> StreamResult<()> { + match producer.kind.as_ref() { + Some(Kind::Workflow(workflow)) => { + if workflow.run_id.is_empty() { + return missing("run id"); + } + workflow.epoch.as_ref().map_or(Ok(()), check_task) + } + Some(Kind::LocalActivity(local)) => { + if local.run_id.is_empty() { + return missing("run id"); + } + let Some(epoch) = local.epoch.as_ref() else { + return missing("epoch"); + }; + let Some(task) = epoch.task.as_ref() else { + return missing("Workflow Task that dispatched it"); + }; + check_task(task)?; + at_least_one(epoch.attempt, "Local Activity") + } + Some(Kind::WorkflowActivity(activity) | Kind::StandaloneActivity(activity)) => { + let Some(epoch) = activity.epoch.as_ref() else { + return missing("epoch"); + }; + if epoch.attempt_scheduled_time.is_none() { + return missing("attempt scheduled time"); + } + at_least_one(epoch.attempt, "Activity") + } + Some(Kind::Client(_)) => Ok(()), + None => missing("kind"), + } +} + +/// The keys of one Stream. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StreamKeys { + base: String, +} + +impl StreamKeys { + /// The keys of `stream_id` under `prefix`. + pub fn new(prefix: &str, stream_id: &str) -> Self { + Self { + base: format!("{}:{{{}}}", encode_part(prefix), encode_part(stream_id)), + } + } + + /// The hash with the format version, the fence, dedupe and close state. + pub fn meta(&self) -> String { + format!("{}:meta", self.base) + } + + /// The log. + pub fn log(&self) -> String { + format!("{}:log", self.base) + } + + /// The key that a close adds one entry to. A close writes no log entry, so a waiting read also + /// waits on this key, and the close wakes it. + pub fn closed(&self) -> String { + format!("{}:closed", self.base) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + Reason, + proto::{ + ActivityEpoch, ActivityProducer, ClientProducer, LocalActivityEpoch, + LocalActivityProducer, WorkflowProducer, WorkflowTaskEpoch, + }, + }; + use prost_types::Timestamp; + + fn producer(kind: ProducerKind, id: &str, run_id: &str) -> Producer { + let run_id = run_id.to_string(); + let kind = match kind { + ProducerKind::Workflow => Kind::Workflow(WorkflowProducer { + run_id, + ..Default::default() + }), + ProducerKind::LocalActivity => Kind::LocalActivity(LocalActivityProducer { + run_id, + ..Default::default() + }), + ProducerKind::Activity => Kind::WorkflowActivity(ActivityProducer { + run_id, + ..Default::default() + }), + ProducerKind::StandaloneActivity => Kind::StandaloneActivity(ActivityProducer { + run_id, + ..Default::default() + }), + ProducerKind::Client => Kind::Client(ClientProducer::default()), + }; + Producer { + id: id.to_string(), + kind: Some(kind), + } + } + + #[test] + fn a_producer_id_is_any_non_empty_text_up_to_the_limit() { + for id in ["x", "wf:a|b", "client:{é} %", "a:b:c:d:e:f"] { + check_producer(&producer(ProducerKind::Client, id, "")).unwrap(); + } + check_producer(&producer( + ProducerKind::Workflow, + &"a".repeat(MAX_PRODUCER_ID_BYTES), + "r", + )) + .unwrap(); + let no_kind = Producer { + kind: None, + ..producer(ProducerKind::Workflow, "wf:x", "x") + }; + let too_long = producer( + ProducerKind::Client, + &"a".repeat(MAX_PRODUCER_ID_BYTES + 1), + "", + ); + let empty = producer(ProducerKind::Client, "", ""); + for bad in [no_kind, too_long, empty] { + assert_eq!( + check_producer(&bad).unwrap_err().reason, + Reason::Invalid, + "{bad:?}" + ); + } + } + + #[test] + fn a_stream_id_is_bounded() { + check_stream_id("a").unwrap(); + check_stream_id(&"a".repeat(MAX_STREAM_ID_BYTES)).unwrap(); + assert!(check_stream_id("").is_err()); + assert!(check_stream_id(&"a".repeat(MAX_STREAM_ID_BYTES + 1)).is_err()); + } + + /// A Local Activity Producer with every field the API requires. + fn local_activity() -> Producer { + Producer { + id: "la:r:k".to_string(), + kind: Some(Kind::LocalActivity(LocalActivityProducer { + run_id: "r".to_string(), + epoch: Some(LocalActivityEpoch { + task: Some(WorkflowTaskEpoch { + started_event_id: 3, + attempt: 1, + started_time: None, + }), + attempt: 1, + }), + })), + } + } + + /// An Activity Producer with every field the API requires. + fn activity() -> Producer { + Producer { + id: "act:r:a:s".to_string(), + kind: Some(Kind::WorkflowActivity(ActivityProducer { + run_id: "r".to_string(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: Some(Timestamp::default()), + attempt: 1, + }), + })), + } + } + + /// A client Producer with every field the API requires. + fn client() -> Producer { + Producer { + id: "client:n:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } + } + + fn local_epoch(producer: &mut Producer) -> &mut LocalActivityEpoch { + let Some(Kind::LocalActivity(local)) = producer.kind.as_mut() else { + unreachable!() + }; + local.epoch.as_mut().unwrap() + } + + fn activity_epoch(producer: &mut Producer) -> &mut ActivityEpoch { + let Some(Kind::WorkflowActivity(activity)) = producer.kind.as_mut() else { + unreachable!() + }; + activity.epoch.as_mut().unwrap() + } + + fn refused(producer: &Producer) -> String { + let error = check_required(producer).unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + error.message + } + + #[test] + fn a_producer_with_its_required_fields_passes() { + // A Workflow Producer only closes, with no epoch, and a standalone Activity has no run. + let workflow = producer(ProducerKind::Workflow, "wf:r", "r"); + let mut standalone = activity(); + let Some(Kind::WorkflowActivity(kind)) = standalone.kind.take() else { + unreachable!() + }; + standalone.kind = Some(Kind::StandaloneActivity(ActivityProducer { + run_id: String::new(), + ..kind + })); + for producer in [workflow, local_activity(), activity(), standalone, client()] { + check_required(&producer).unwrap(); + } + } + + #[test] + fn a_workflow_producer_needs_its_run_id() { + let workflow = producer(ProducerKind::Workflow, "wf:", ""); + assert!(refused(&workflow).contains("run id")); + } + + /// A Workflow Producer with `epoch`. + fn workflow_with(epoch: WorkflowTaskEpoch) -> Producer { + Producer { + id: "wf:r".to_string(), + kind: Some(Kind::Workflow(WorkflowProducer { + run_id: "r".to_string(), + epoch: Some(epoch), + })), + } + } + + #[test] + fn a_workflow_epoch_passes_with_its_required_fields() { + let epoch = WorkflowTaskEpoch { + started_event_id: 3, + attempt: 1, + started_time: None, + }; + check_required(&workflow_with(epoch)).unwrap(); + } + + #[test] + fn a_workflow_epoch_needs_its_started_event_id() { + let epoch = WorkflowTaskEpoch { + attempt: 1, + ..Default::default() + }; + assert!(refused(&workflow_with(epoch)).contains("started event id")); + } + + #[test] + fn a_workflow_epoch_attempt_is_1_or_more() { + let epoch = WorkflowTaskEpoch { + started_event_id: 3, + ..Default::default() + }; + assert!(refused(&workflow_with(epoch)).contains("Workflow Task attempt")); + } + + #[test] + fn a_local_activity_producer_needs_its_run_id() { + let mut local = local_activity(); + let Some(Kind::LocalActivity(kind)) = local.kind.as_mut() else { + unreachable!() + }; + kind.run_id.clear(); + assert!(refused(&local).contains("run id")); + } + + #[test] + fn a_local_activity_producer_needs_its_epoch() { + let mut local = local_activity(); + let Some(Kind::LocalActivity(kind)) = local.kind.as_mut() else { + unreachable!() + }; + kind.epoch = None; + assert!(refused(&local).contains("epoch")); + } + + #[test] + fn a_local_activity_epoch_needs_its_workflow_task() { + let mut local = local_activity(); + local_epoch(&mut local).task = None; + assert!(refused(&local).contains("Workflow Task")); + } + + #[test] + fn a_local_activity_task_needs_its_started_event_id() { + let mut local = local_activity(); + local_epoch(&mut local) + .task + .as_mut() + .unwrap() + .started_event_id = 0; + assert!(refused(&local).contains("Workflow Task")); + } + + #[test] + fn a_local_activity_task_attempt_is_1_or_more() { + let mut local = local_activity(); + local_epoch(&mut local).task.as_mut().unwrap().attempt = 0; + assert!(refused(&local).contains("Workflow Task attempt")); + } + + #[test] + fn a_local_activity_attempt_is_1_or_more() { + let mut local = local_activity(); + local_epoch(&mut local).attempt = 0; + assert!(refused(&local).contains("Local Activity attempt")); + } + + #[test] + fn an_activity_producer_needs_its_epoch() { + let mut act = activity(); + let Some(Kind::WorkflowActivity(kind)) = act.kind.as_mut() else { + unreachable!() + }; + kind.epoch = None; + assert!(refused(&act).contains("epoch")); + } + + #[test] + fn an_activity_epoch_needs_its_scheduled_time() { + let mut act = activity(); + activity_epoch(&mut act).attempt_scheduled_time = None; + assert!(refused(&act).contains("scheduled time")); + } + + #[test] + fn an_activity_attempt_is_1_or_more() { + let mut act = activity(); + activity_epoch(&mut act).attempt = 0; + assert!(refused(&act).contains("Activity attempt")); + } +} diff --git a/crates/streams/src/lib.rs b/crates/streams/src/lib.rs new file mode 100644 index 000000000..fffcb54c2 --- /dev/null +++ b/crates/streams/src/lib.rs @@ -0,0 +1,35 @@ +#![warn(missing_docs)] + +//! The Stream runtime of Temporal SDKs. It is experimental. +//! +//! The crate serves the five operations of the Stream service on a Stream Store. A lang bridge +//! builds the runtime and calls it with an operation name and the encoded request. + +pub mod digest; +mod dispatch; +pub mod epoch; +mod error; +pub mod ids; +pub mod memory; +mod metrics; +pub mod offset; +mod read; +mod record; +mod runtime; +pub mod store; +#[cfg(any(test, feature = "test-support"))] +pub mod testing; + +pub use error::{Reason, StreamError, StreamResult}; +pub use read::MAX_WAIT; +pub use runtime::{MemoryStreamOptions, MemoryStreamRuntime, StreamRuntime}; + +/// The protos that lang and the crate exchange. `temporalio-protos` generates them, so a value +/// needs no conversion between the crates. +pub mod proto { + pub use temporalio_common::protos::temporal::api::{ + enums::v1::{StreamErrorReason, StreamIdConflictPolicy, StreamStatus}, + stream::v1::*, + streamservice::v1::*, + }; +} diff --git a/crates/streams/src/memory.rs b/crates/streams/src/memory.rs new file mode 100644 index 000000000..d9f1356dd --- /dev/null +++ b/crates/streams/src/memory.rs @@ -0,0 +1,711 @@ +//! The memory Store, for tests. It keeps the same fields as the Redis layout and runs the steps of +//! the Lua scripts in the same order, so that both Stores return the same results. Each call holds +//! one lock, since Redis runs one script at a time. + +use crate::{ + Reason, StreamError, StreamResult, + digest::DIGEST_LEN, + epoch::{EPOCH_KEY_LEN, ProducerKind, epoch_key}, + ids::StreamKeys, + offset::{entry_id_of_offset, offset_of_entry_id}, + proto::{Producer, StreamRef}, + store::{ + Appended, Batch, Capabilities, Closed, ConflictPolicy, Create, CreateOutcome, Created, + Entry, FORMAT_VERSION, MAX_SEQUENCE, Store, StreamState, TOMBSTONE, + }, +}; +use parking_lot::Mutex; +use std::{ + collections::{BTreeMap, HashMap}, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::{sync::Notify, time::Instant}; + +/// The number of sequence numbers in one millisecond, since an offset packs them in 20 bits. +const SEQ_LIMIT: u64 = 1 << crate::offset::SEQ_BITS; + +#[derive(Clone, Default)] +struct Fields { + record: Option>, +} + +/// One Stream's meta hash and log, each with an expiry time, as Redis keys have. +#[derive(Default)] +struct Keys { + /// The meta hash, whose field names and values are bytes as in Redis. + meta: HashMap, Vec>, + meta_expires: i64, + /// `None` when the log key doesn't exist. + log: Option>, + log_expires: i64, + /// The newest id of the log, which Redis drops with the expired key. + last_id: (u64, u64), +} + +/// A Store in process memory, with the rules of the Redis scripts. +pub struct MemoryStore { + prefix: String, + retention: Duration, + last_write_ttl: Option, + streams: Mutex>, + appended: Notify, +} + +impl std::fmt::Debug for MemoryStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MemoryStore") + .field("retention", &self.retention) + .field("last_write_ttl", &self.last_write_ttl) + .finish_non_exhaustive() + } +} + +fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |since| since.as_millis() as i64) +} + +fn millis(duration: Duration) -> i64 { + i64::try_from(duration.as_millis()).unwrap_or(i64::MAX) +} + +fn refuse(reason: Reason, message: &str) -> StreamError { + StreamError::new(reason, message) +} + +fn id_text((ms, seq): (u64, u64)) -> String { + format!("{ms}-{seq}") +} + +fn parse_id(text: &str) -> Option<(u64, u64)> { + let (ms, seq) = text.split_once('-')?; + Some((ms.parse().ok()?, seq.parse().ok()?)) +} + +/// The length of a [ProducerField] value. +const PRODUCER_FIELD_LEN: usize = EPOCH_KEY_LEN + 8 + 2 * 12 + DIGEST_LEN; + +/// An entry id as a [ProducerField] keeps it, the ms and the seq. +pub(crate) type EntryId = (u64, u32); + +/// The `p||` field, one per Producer. Its value is 88 bytes. They hold the fence +/// and the last batch's sequence (8), first and last entry ids (12 each, the ms in 8 and the seq in +/// 4, zeros for none) and digest. Numbers are big-endian. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ProducerField { + /// The newest epoch key of the Producer, which is also the epoch of its last batch. + pub(crate) fence: [u8; EPOCH_KEY_LEN], + /// The last batch's sequence. + pub(crate) sequence: i64, + /// The last batch's first and last entry ids, `None` when it wrote no entry. + pub(crate) entries: Option<(EntryId, EntryId)>, + /// The last batch's digest. + pub(crate) digest: [u8; DIGEST_LEN], +} + +impl ProducerField { + pub(crate) fn encode(&self) -> Vec { + let mut value = Vec::with_capacity(PRODUCER_FIELD_LEN); + value.extend_from_slice(&self.fence); + value.extend_from_slice(&self.sequence.to_be_bytes()); + let (first, last) = self.entries.unwrap_or_default(); + for (ms, seq) in [first, last] { + value.extend_from_slice(&ms.to_be_bytes()); + value.extend_from_slice(&seq.to_be_bytes()); + } + value.extend_from_slice(&self.digest); + value + } +} + +/// Refuses a value of another length, which a later format may write. +impl TryFrom<&[u8]> for ProducerField { + type Error = StreamError; + + fn try_from(value: &[u8]) -> StreamResult { + let unknown = || { + refuse( + Reason::StorageCorrupt, + "the Producer field has an unknown format", + ) + }; + if value.len() != PRODUCER_FIELD_LEN { + return Err(unknown()); + } + let (fence, rest) = value.split_at(EPOCH_KEY_LEN); + let (sequence, rest) = rest.split_at(8); + let (first, rest) = rest.split_at(12); + let (last, digest) = rest.split_at(12); + let entry_id = |bytes: &[u8]| -> StreamResult { + let (ms, seq) = bytes.split_at(8); + let ms = ms.try_into().map_err(|_| unknown())?; + let seq = seq.try_into().map_err(|_| unknown())?; + Ok((u64::from_be_bytes(ms), u32::from_be_bytes(seq))) + }; + // A batch has both entry ids or neither. + let entries = match (entry_id(first)?, entry_id(last)?) { + ((0, 0), (0, 0)) => None, + ((0, 0), _) | (_, (0, 0)) => return Err(unknown()), + ids => Some(ids), + }; + Ok(Self { + fence: fence.try_into().map_err(|_| unknown())?, + sequence: i64::from_be_bytes(sequence.try_into().map_err(|_| unknown())?), + entries, + digest: digest.try_into().map_err(|_| unknown())?, + }) + } +} + +/// The [EntryId] of an entry id's text, `-`. +fn entry_id(text: &str) -> StreamResult { + let bad = || refuse(Reason::StorageCorrupt, "an entry id isn't -"); + let (ms, seq) = text.split_once('-').ok_or_else(bad)?; + Ok(( + ms.parse().map_err(|_| bad())?, + seq.parse().map_err(|_| bad())?, + )) +} + +impl Keys { + fn get(&self, field: impl AsRef<[u8]>) -> Option<&[u8]> { + self.meta.get(field.as_ref()).map(Vec::as_slice) + } + + /// A field that holds text, such as `newest`. `None` when it's absent or not text. + fn text(&self, field: impl AsRef<[u8]>) -> Option<&str> { + self.get(field) + .and_then(|value| std::str::from_utf8(value).ok()) + } + + /// A field that holds a number, zero when it's absent. + fn number(&self, field: &str) -> i64 { + self.text(field) + .and_then(|number| number.parse().ok()) + .unwrap_or(0) + } + + fn set(&mut self, field: impl Into>, value: impl Into>) { + self.meta.insert(field.into(), value.into()); + } + + fn remove(&mut self, field: impl AsRef<[u8]>) { + self.meta.remove(field.as_ref()); + } + + /// The `check_live` step, which checks the create key, the format version and the expiry. + fn check_live(&self, now: i64) -> StreamResult<()> { + if self.get("v").is_none() { + return Err(refuse(Reason::NotFound, "nobody created the Stream")); + } + let v = self.number("v"); + if v > i64::from(FORMAT_VERSION) { + return Err(refuse( + Reason::FormatTooNew, + &format!("format {v} is newer than {FORMAT_VERSION}"), + )); + } + if self + .text("exp") + .and_then(|exp| exp.parse::().ok()) + .is_some_and(|exp| exp <= now) + { + return Err(refuse(Reason::Expired, "the Stream expired")); + } + Ok(()) + } + + fn refresh(&mut self, now: i64, last_write_ttl: Option, tombstone: i64) { + // An open Stream keeps the last-write TTL after its last write, and with none it stays + // until its close. A closed one keeps the Retention. + let ttl = if self.get("closed").is_none() { + last_write_ttl + } else { + Some(self.number("ret")) + }; + let Some(ttl) = ttl else { + self.remove("exp"); + self.meta_expires = 0; + self.log_expires = 0; + return; + }; + let exp = now.saturating_add(ttl); + self.set("exp", exp.to_string()); + self.meta_expires = exp.saturating_add(tombstone); + if self.log.is_some() { + self.log_expires = exp; + } + } + + /// The newest id the Stream gave. Redis drops it with an expired log, so the meta's `newest` + /// keeps the next ids above the old ones. + fn newest_id(&self) -> (u64, u64) { + let newest = self.text("newest").and_then(parse_id); + self.last_id.max(newest.unwrap_or_default()) + } + + /// Raises the trimmed offset to `newest` when the log is gone, as `mark_log_gone` in `lib.lua`. + /// The log took every Record with it, so a read from an older offset must fail. + fn mark_log_gone(&mut self) { + if self.log.is_some() { + return; + } + if let Some(newest) = self.text("newest").and_then(parse_id) { + self.raise_trimmed(newest); + } + } + + /// Sets the trimmed offset to `id`, unless it is already at or past it. + fn raise_trimmed(&mut self, id: (u64, u64)) { + let moves = self + .text("trimmed") + .and_then(parse_id) + .is_none_or(|trimmed| trimmed < id); + if moves { + self.set("trimmed", id_text(id)); + } + } + + /// Refuses `count` entries whose ids would not fit an offset. The ids go on from the newest id + /// the Stream gave, as `check_ids_fit` in `lib.lua`. + fn check_ids_fit(&self, now: i64, count: u64) -> StreamResult<()> { + let (ms, seq) = self.newest_id(); + let first = if now > ms as i64 { 0 } else { seq + 1 }; + if first + count > SEQ_LIMIT { + return Err(refuse( + Reason::StorageUnavailable, + "the entry ids of this millisecond would not fit an offset", + )); + } + Ok(()) + } + + /// Adds an entry as `add` in `lib.lua` does, with the next id after the newest one, in this + /// millisecond or a later one. + fn add(&mut self, now: i64, fields: Fields) -> String { + let now = now.max(0) as u64; + let (last_ms, last_seq) = self.newest_id(); + let id = if now > last_ms { + (now, 0) + } else { + (last_ms, last_seq + 1) + }; + self.last_id = id; + self.log + .get_or_insert_with(BTreeMap::new) + .insert(id, fields); + id_text(id) + } +} + +impl MemoryStore { + /// An empty Store. + pub fn new(retention: Duration, last_write_ttl: Option) -> Self { + Self { + prefix: crate::ids::DEFAULT_KEY_PREFIX.to_string(), + retention, + last_write_ttl, + streams: Mutex::default(), + appended: Notify::new(), + } + } + + fn key(&self, stream_id: &str) -> String { + StreamKeys::new(&self.prefix, stream_id).meta() + } + + /// Runs `step` on one Stream's keys after it drops the expired keys, as Redis does. + fn with(&self, stream_id: &str, step: impl FnOnce(&mut Keys, i64) -> T) -> T { + let now = now_ms(); + let mut streams = self.streams.lock(); + let key = self.key(stream_id); + let keys = streams.entry(key.clone()).or_default(); + if keys.meta_expires > 0 && keys.meta_expires <= now { + keys.meta.clear(); + keys.meta_expires = 0; + } + if keys.log_expires > 0 && keys.log_expires <= now { + keys.log = None; + keys.log_expires = 0; + keys.last_id = (0, 0); + } + let result = step(keys, now); + // Once every key is gone, Redis has no state of the Stream, not even its newest id. + if keys.meta.is_empty() && keys.log.is_none() { + streams.remove(&key); + } + result + } +} + +#[cfg(any(test, feature = "test-support"))] +impl MemoryStore { + /// Changes one meta field, for tests of what the scripts refuse. + #[doc(hidden)] + pub fn test_set_meta(&self, stream_id: &str, field: &[u8], value: &[u8]) { + self.with(stream_id, |keys, _| keys.set(field, value)); + } + + /// One meta field, for tests. + #[doc(hidden)] + pub fn test_meta(&self, stream_id: &str, field: &[u8]) -> Option> { + self.with(stream_id, |keys, _| keys.get(field).map(<[u8]>::to_vec)) + } + + /// The ms left on the log and on the meta, -1 for a key with no expiry and -2 for a key that + /// doesn't exist, as `PTTL`. + #[doc(hidden)] + pub fn test_ttls(&self, stream_id: &str) -> (i64, i64) { + self.with(stream_id, |keys, now| { + let ttl = |expires: i64| if expires > 0 { expires - now } else { -1 }; + let log = if keys.log.is_some() { + ttl(keys.log_expires) + } else { + -2 + }; + let meta = if keys.meta.is_empty() { + -2 + } else { + ttl(keys.meta_expires) + }; + (log, meta) + }) + } + + /// Adds an entry with an explicit id, as `XADD `, for tests. + #[doc(hidden)] + pub fn test_add_entry(&self, stream_id: &str, id: &str) { + self.with(stream_id, |keys, _| { + let id = parse_id(id).expect("an entry id"); + keys.last_id = keys.last_id.max(id); + keys.log + .get_or_insert_with(BTreeMap::new) + .insert(id, Fields::default()); + }); + } + + /// The number of log entries, as `XLEN`, for tests. + #[doc(hidden)] + pub fn test_log_len(&self, stream_id: &str) -> usize { + self.with(stream_id, |keys, _| { + keys.log.as_ref().map_or(0, BTreeMap::len) + }) + } + + /// Deletes the log, as `DEL`, for tests. + #[doc(hidden)] + pub fn test_drop_log(&self, stream_id: &str) { + self.with(stream_id, |keys, _| { + keys.log = None; + keys.log_expires = 0; + keys.last_id = (0, 0); + }); + } +} + +fn offsets(first: &str, last: &str) -> StreamResult> { + if first.is_empty() || last.is_empty() { + return Ok(None); + } + Ok(Some(( + offset_of_entry_id(first)?, + offset_of_entry_id(last)?, + ))) +} + +#[async_trait::async_trait] +impl Store for MemoryStore { + fn capabilities(&self) -> Capabilities { + Capabilities::ALL + } + + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult { + let stream_id = stream_ref.id.as_str(); + let retention = create.retention.filter(|r| !r.is_zero()); + let retention = millis(retention.unwrap_or(self.retention)); + let request_id = create.request_id; + let (last_write_ttl, tombstone) = (self.last_write_ttl.map(millis), millis(TOMBSTONE)); + let created = self.with(stream_id, |keys, now| { + if keys.get("ret").is_none() { + keys.set("v", FORMAT_VERSION.to_string()); + keys.set("ret", retention.to_string()); + if !request_id.is_empty() { + keys.set("rid", request_id); + } + keys.refresh(now, last_write_ttl, tombstone); + return Ok((CreateOutcome::Created, retention)); + } + keys.check_live(now)?; + if keys.get("closed").is_some() { + return Err(refuse(Reason::Closed, "the Stream is closed")); + } + // A retry of the create that made the Stream gets its answer again. + if !request_id.is_empty() && keys.get("rid") == Some(request_id.as_bytes()) { + return Ok((CreateOutcome::Created, keys.number("ret"))); + } + if create.conflict == ConflictPolicy::Fail { + return Err(refuse(Reason::AlreadyExists, "the Stream exists")); + } + Ok((CreateOutcome::Exists, keys.number("ret"))) + }); + let (outcome, stored) = created?; + Ok(Created { + outcome, + retention: Duration::from_millis(stored.max(0) as u64), + }) + } + + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult { + let stream_id = stream_ref.id.as_str(); + let producer = batch.producer; + let kind = ProducerKind::try_from(producer)?; + let epoch = epoch_key(producer.kind.as_ref()); + let (last_write_ttl, tombstone) = (self.last_write_ttl.map(millis), millis(TOMBSTONE)); + let field = format!("p|{}|{}", kind as i32, producer.id); + let appended = self.with(stream_id, |keys, now| { + // The format version comes first, since a newer format can change the Producer field. + keys.check_live(now)?; + let stored = keys.get(&field).map(ProducerField::try_from).transpose()?; + // Every append sets the fence to its epoch, so the fence is the last batch's epoch. + let previous = stored.filter(|stored| stored.fence == epoch); + // The lock also covers the dedupe check, so no other batch takes the same number. + let sequence = match batch.sequence { + 0 => previous.map_or(1, |previous| previous.sequence + 1), + sequence => sequence, + }; + if !(1..=MAX_SEQUENCE).contains(&sequence) { + return Err(refuse( + Reason::Invalid, + "the sequence must be from 1 to 2^53 - 1", + )); + } + if let Some(previous) = previous { + if sequence == previous.sequence { + if previous.digest != batch.digest { + return Err(refuse( + Reason::SeqDivergent, + &format!( + "sequence {sequence} has other content than the batch the Store \ + has" + ), + )); + } + let (first, last) = previous + .entries + .map(|((ms, seq), (last_ms, last_seq))| { + (format!("{ms}-{seq}"), format!("{last_ms}-{last_seq}")) + }) + .unwrap_or_default(); + return Ok((true, first, last)); + } + if sequence < previous.sequence { + return Err(refuse( + Reason::SequenceTooLow, + &format!( + "sequence {sequence} is below the last one, {}", + previous.sequence + ), + )); + } + } + if keys.get("closed").is_some() { + return Err(refuse(Reason::Closed, "the Stream is closed")); + } + if stored.is_some_and(|stored| epoch < stored.fence) { + return Err(refuse( + Reason::Fenced, + "the epoch is lower than the newest one of the Producer", + )); + } + let records = batch.records.len() as u64; + keys.check_ids_fit(now, records)?; + + keys.mark_log_gone(); + let (mut first, mut last) = (String::new(), String::new()); + for record in batch.records { + last = keys.add( + now, + Fields { + record: Some(record.clone()), + }, + ); + if first.is_empty() { + first = last.clone(); + } + } + let entries = if last.is_empty() { + None + } else { + Some((entry_id(&first)?, entry_id(&last)?)) + }; + let value = ProducerField { + fence: epoch, + sequence, + entries, + digest: batch.digest, + }; + keys.set(field.as_str(), value.encode()); + if !last.is_empty() { + keys.set("newest", last.clone()); + } + keys.refresh(now, last_write_ttl, tombstone); + Ok((false, first, last)) + }); + let (resend, first, last) = appended?; + if !resend && !last.is_empty() { + self.appended.notify_waiters(); + } + Ok(Appended { + offsets: offsets(&first, &last)?, + resend, + }) + } + + async fn close(&self, stream_ref: &StreamRef, producer: &Producer) -> StreamResult { + let stream_id = stream_ref.id.as_str(); + let kind = ProducerKind::try_from(producer)?; + let epoch = epoch_key(producer.kind.as_ref()); + let field = format!("p|{}|{}", kind as i32, producer.id); + let tombstone = millis(TOMBSTONE); + let closed = self.with(stream_id, |keys, now| { + keys.check_live(now)?; + if keys.get("closed").is_some() { + return Ok(true); + } + let stored = keys.get(&field).map(ProducerField::try_from).transpose()?; + if stored.is_some_and(|stored| epoch < stored.fence) { + return Err(refuse( + Reason::Fenced, + "the epoch is lower than the newest one of the Producer", + )); + } + keys.set("closed", "1"); + keys.refresh(now, None, tombstone); + Ok(false) + }); + let already = closed?; + // No entry wakes a waiting read, so the close wakes it. + if !already { + self.appended.notify_waiters(); + } + Ok(Closed { already }) + } + + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)> { + let stream_id = stream_ref.id.as_str(); + let start = if from > 0 { + parse_id(&entry_id_of_offset(from)).unwrap_or((0, 0)) + } else { + (0, 0) + }; + self.with(stream_id, |keys, now| { + let offset = |field: &str| -> StreamResult { + keys.text(field).map_or(Ok(0), offset_of_entry_id) + }; + let number = |field: &str| keys.number(field); + let state = StreamState { + exists: keys.get("v").is_some(), + version: number("v") as u32, + closed: keys.get("closed").is_some(), + newest: offset("newest")?, + trimmed: offset("trimmed")?, + expires_ms: number("exp"), + now_ms: now, + }; + let mut entries = vec![]; + if let Some(log) = keys.log.as_ref() { + for (id, fields) in log.range(start..).take(count) { + entries.push(Entry { + offset: offset_of_entry_id(&id_text(*id))?, + record: fields.record.clone(), + }); + } + } + Ok((state, entries)) + }) + } + + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult { + let stream_id = stream_ref.id.as_str(); + let after = if after > 0 { + parse_id(&entry_id_of_offset(after)).unwrap_or((0, 0)) + } else { + (0, 0) + }; + loop { + // Listen before the check, so an append between the two still wakes the wait. + let appended = self.appended.notified(); + let found = self.with(stream_id, |keys, _| { + keys.get("closed").is_some() + || keys + .log + .as_ref() + .and_then(|log| log.keys().next_back()) + .is_some_and(|id| *id > after) + }); + if found { + return Ok(true); + } + if tokio::time::timeout_at(deadline, appended).await.is_err() { + return Ok(false); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_id_follows_the_newest_one_within_a_millisecond() { + let mut keys = Keys::default(); + assert_eq!(keys.add(5, Fields::default()), "5-0"); + assert_eq!(keys.add(5, Fields::default()), "5-1"); + // The ids keep increasing when the clock steps back. + assert_eq!(keys.add(3, Fields::default()), "5-2"); + assert_eq!(keys.add(9, Fields::default()), "9-0"); + } + + #[test] + fn a_producer_field_matches_the_shared_vectors() { + let vectors: serde_json::Value = + serde_json::from_str(include_str!("../testdata/fields.json")).unwrap(); + let hex = |text: &str| -> Vec { + (0..text.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&text[i..i + 2], 16).unwrap()) + .collect() + }; + for case in vectors["producer_values"].as_array().unwrap() { + let text = |name: &str| case[name].as_str().unwrap(); + let key = |name: &str| case[name].as_str().map(|key| hex(key).try_into().unwrap()); + let entries = (!text("first").is_empty()).then(|| { + ( + entry_id(text("first")).unwrap(), + entry_id(text("last")).unwrap(), + ) + }); + let field = ProducerField { + fence: key("fence").unwrap(), + sequence: case["sequence"].as_i64().unwrap(), + entries, + digest: hex(text("digest")).try_into().unwrap(), + }; + let value = hex(text("value")); + assert_eq!(field.encode(), value, "{}", text("name")); + assert_eq!(ProducerField::try_from(value.as_slice()).unwrap(), field); + } + } +} diff --git a/crates/streams/src/metrics.rs b/crates/streams/src/metrics.rs new file mode 100644 index 000000000..5fa5fb14d --- /dev/null +++ b/crates/streams/src/metrics.rs @@ -0,0 +1,51 @@ +//! The crate's metrics. They use the meter that lang gives, or a no-op meter. + +use temporalio_common::telemetry::metrics::{ + Counter, MetricAttributes, MetricParameters, TemporalMeter, UpDownCounter, +}; + +#[derive(Clone)] +pub(crate) struct Metrics { + attributes: MetricAttributes, + fenced: Counter, + blocking_reads_waiting: UpDownCounter, +} + +fn parameters( + name: &'static str, + description: &'static str, + unit: &'static str, +) -> MetricParameters { + MetricParameters { + name: name.into(), + description: description.into(), + unit: unit.into(), + } +} + +impl Metrics { + pub(crate) fn new(meter: &TemporalMeter) -> Self { + Self { + attributes: meter.get_default_attributes().clone(), + fenced: meter.counter(parameters( + "stream_fenced", + "Batches that the Store refused for a lower epoch", + "", + )), + blocking_reads_waiting: meter.up_down_counter(parameters( + "stream_blocking_reads_waiting", + "Reads that wait on the Store", + "", + )), + } + } + + pub(crate) fn fenced(&self) { + self.fenced.add(1, &self.attributes); + } + + pub(crate) fn read_waits(&self, waiting: bool) { + self.blocking_reads_waiting + .add(if waiting { 1 } else { -1 }, &self.attributes); + } +} diff --git a/crates/streams/src/offset.rs b/crates/streams/src/offset.rs new file mode 100644 index 000000000..4b3f580b7 --- /dev/null +++ b/crates/streams/src/offset.rs @@ -0,0 +1,101 @@ +//! Offsets. An offset is the Redis entry id `-` packed as `ms << 20 | seq` in an `i64`. +//! It is ordered and unique in one Stream, and sparse. Zero is never an offset. +//! +//! On the wire the offset is opaque `bytes`. Lang only compares two offsets and resumes a read with +//! one, so the crate keeps the `i64` form inside and converts at the proto boundary with +//! [offset_to_wire] and [offset_from_wire]. + +use crate::{StreamError, StreamResult}; + +/// The number of bits of the entry id's sequence part. +pub const SEQ_BITS: u32 = 20; + +/// The wire form of an offset, for the read and append fields that lang reads as opaque `bytes`. +/// +/// Zero, which is "no offset", is the empty string, so it compares below every real offset. A real +/// offset is 8-byte big-endian, which keeps the order of the positive `i64` values. +pub fn offset_to_wire(offset: i64) -> Vec { + if offset <= 0 { + return Vec::new(); + } + (offset as u64).to_be_bytes().to_vec() +} + +/// The offset of a wire value that [offset_to_wire] wrote. The empty string is "no offset", zero. +/// +/// A value longer than 8 bytes is a bad request, since no offset has more. +pub fn offset_from_wire(bytes: &[u8]) -> StreamResult { + if bytes.is_empty() { + return Ok(0); + } + if bytes.len() > 8 { + return Err(StreamError::invalid(format!( + "A Stream offset has {} bytes, more than 8", + bytes.len() + ))); + } + let mut buf = [0u8; 8]; + buf[8 - bytes.len()..].copy_from_slice(bytes); + Ok(u64::from_be_bytes(buf) as i64) +} + +const SEQ_MASK: i64 = (1 << SEQ_BITS) - 1; +/// The largest millisecond part that fits, which is past the year 2200. +const MAX_MS: u64 = (i64::MAX >> SEQ_BITS) as u64; + +/// Packs a Redis entry id. +pub fn offset_of_entry_id(id: &str) -> StreamResult { + let bad = || StreamError::storage_corrupt(format!("The Store gave the entry id {id:?}")); + let (ms, seq) = id.split_once('-').ok_or_else(bad)?; + let ms: u64 = ms.parse().map_err(|_| bad())?; + let seq: u64 = seq.parse().map_err(|_| bad())?; + if ms > MAX_MS || seq > SEQ_MASK as u64 { + return Err(StreamError::storage_corrupt(format!( + "The entry id {id:?} doesn't fit in an offset" + ))); + } + Ok(((ms as i64) << SEQ_BITS) | seq as i64) +} + +/// Unpacks an offset into its Redis entry id. The offset must be above zero. +pub fn entry_id_of_offset(offset: i64) -> String { + debug_assert!(offset > 0); + format!("{}-{}", offset >> SEQ_BITS, offset & SEQ_MASK) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_wire_offset_keeps_the_order_and_empty_is_none() { + assert_eq!(offset_to_wire(0), Vec::::new()); + assert_eq!(offset_from_wire(&[]).unwrap(), 0); + for offset in [1, 42, (1 << SEQ_BITS) | 7, i64::MAX] { + assert_eq!(offset_from_wire(&offset_to_wire(offset)).unwrap(), offset); + } + // The byte order matches the i64 order, so lang compares two offsets as bytes. + assert!(offset_to_wire(1) > offset_to_wire(0)); + assert!(offset_to_wire(256) > offset_to_wire(255)); + // A right-aligned short value reads as the same number a full one would. + assert_eq!(offset_from_wire(&[1]).unwrap(), 1); + assert_eq!( + offset_from_wire(&[0, 0, 0, 0, 0, 0, 0, 0, 0]) + .unwrap_err() + .reason, + crate::Reason::Invalid + ); + } + + #[test] + fn an_entry_id_that_doesnt_fit_is_refused() { + for id in ["1-1048576", &format!("{}-0", MAX_MS + 1), "x"] { + let error = offset_of_entry_id(id).unwrap_err(); + assert_eq!(error.reason, crate::Reason::StorageCorrupt, "{id}"); + } + assert_eq!( + offset_of_entry_id(&format!("{MAX_MS}-{SEQ_MASK}")).unwrap(), + i64::MAX + ); + } +} diff --git a/crates/streams/src/read.rs b/crates/streams/src/read.rs new file mode 100644 index 000000000..31f9b47db --- /dev/null +++ b/crates/streams/src/read.rs @@ -0,0 +1,216 @@ +//! The read of the runtime, a long poll that pages through the log. + +use crate::{ + Reason, StreamError, StreamResult, StreamRuntime, + offset::{offset_from_wire, offset_to_wire}, + proto::{ReadRecordsRequest, ReadRecordsResponse, StreamRecord, StreamRef}, + runtime::{duration, stream_id}, + store::{FORMAT_VERSION, StreamState}, +}; +use prost::Message; +use std::time::Duration; +use tokio::time::Instant; + +/// The longest wait of one read for new Records. +pub const MAX_WAIT: Duration = Duration::from_secs(60); +/// The maximum number of Records in one read. +const MAX_RECORDS: usize = 1000; + +impl StreamRuntime { + /// A long poll that returns the Records it found, or nothing after `wait`. On a closed Stream + /// with no Record left, it returns `done` at once. + pub async fn read_records( + &self, + request: ReadRecordsRequest, + ) -> StreamResult { + stream_id(request.r#ref.as_ref())?; + let stream = request.r#ref.clone().unwrap_or_default(); + // The wire from_offset is exclusive, so a read starts after it. The empty value is a read + // from the start. + let from_offset = offset_from_wire(&request.from_offset)?; + if from_offset < 0 { + return Err(StreamError::invalid("The offset is below zero")); + } + let max_bytes = max_bytes(request.max_bytes)?; + let max = match request.max_records { + n if n <= 0 => MAX_RECORDS, + n => (n as usize).min(MAX_RECORDS), + }; + let started = Instant::now(); + let wait = duration(request.wait).unwrap_or_default().min(MAX_WAIT); + let deadline = started + wait; + + // A creator hands out the ref only after its create, so a read never waits for one. + let (mut state, _) = self.store.read(&stream, 0, 0).await?; + if !state.exists { + return Err(StreamError::new( + Reason::NotFound, + "Nobody created the Stream", + )); + } + if state.version > FORMAT_VERSION { + return Err(StreamError::new( + Reason::FormatTooNew, + format!("format {} is newer than {FORMAT_VERSION}", state.version), + )); + } + // Before the start is one past the offset, since the largest offset has no next one. + if from_offset > state.newest { + return Err(StreamError::new( + Reason::OffsetTooHigh, + "The offset is past the newest Record. A failover may have lost the newest \ + Records. Resume from the newest offset, or read again once newer Records pass it", + )); + } + // An exclusive from_offset reads after that Record, so the inclusive start is one past it. + let from = match from_offset { + 0 => 0, + offset => offset + 1, + }; + loop { + check_expiry(&state, from)?; + if let Some(page) = self.page(&stream, &state, from, (max, max_bytes)).await? { + return Ok(page); + } + if deadline <= Instant::now() { + // An empty read resumes from where it started, which is one before the inclusive + // start. + return Ok(ReadRecordsResponse { + last_offset: offset_to_wire(from - 1), + ..Default::default() + }); + } + let after = (from - 1).max(state.newest); + self.metrics.read_waits(true); + let woke = self.store.wait_after(&stream, after, deadline).await; + self.metrics.read_waits(false); + woke?; + state = self.store.read(&stream, 0, 0).await?.0; + } + } + + /// The Records from `from`, or `None` when there is none yet and the Stream is open. A page + /// of a closed Stream that gives its last Record is done, also with no Record. The pair + /// `(max, max_bytes)` caps the Records and their bytes. + async fn page( + &self, + stream: &StreamRef, + state: &StreamState, + from: i64, + (max, max_bytes): (usize, usize), + ) -> StreamResult> { + // A read from the start skips the Records that the log lost. A read from an offset fails on + // them, since the Reader asked for those Records. + let retained = state.retained_from(); + let mut start = if from > 0 { from } else { retained }; + let mut records = vec![]; + let mut full = false; + let mut bytes = 0; + // The bytes and the entries that the page read from the Store. + let mut pulled = (0, 0); + // Whether the Stream was closed, and its newest offset, at the newest Store read. + let mut end = (state.closed, state.newest); + while records.len() < max && !full { + let room = (max_bytes != usize::MAX).then(|| max_bytes.saturating_sub(bytes)); + let chunk = chunk_len(max - records.len(), room, pulled); + let (chunk_state, entries) = self.store.read(stream, start, chunk).await?; + // Records that the log lost after the state read leave a gap before these entries. + check_expiry(&chunk_state, if from > 0 { start } else { 0 })?; + end = (chunk_state.closed, chunk_state.newest); + if entries.is_empty() { + // Only an expired log has no entry at an offset up to `newest`. + if from > 0 && start == from && from <= chunk_state.newest { + return Err(expired()); + } + break; + } + for entry in entries { + if entry.offset < retained { + if from > 0 { + return Err(expired()); + } + start = entry.offset + 1; + continue; + } + let Some(encoded) = entry.record else { + start = entry.offset + 1; + continue; + }; + let record = decode_record(&encoded, entry.offset)?; + // An entry over the caps is still returned alone, so that the read always makes + // progress. The next read starts at the entry that didn't fit. + let size = record.encoded_len(); + pulled = (pulled.0 + size, pulled.1 + 1); + let over = bytes + size > max_bytes || records.len() + 1 > max; + if !records.is_empty() && over { + full = true; + break; + } + bytes += size; + records.push(record); + start = entry.offset + 1; + if records.len() >= max { + break; + } + } + } + // A closed Stream takes no more appends, so no Record is left past its newest offset. + let done = end.0 && start > end.1; + if records.is_empty() && !done { + return Ok(None); + } + // The resume is after the last Record returned, which is one before the inclusive start. + let page = ReadRecordsResponse { + records, + last_offset: offset_to_wire(start - 1), + done, + }; + Ok(Some(page)) + } +} + +/// The number of entries of the next Store read, at most `left`. With a byte cap, the first read +/// takes one entry, and each later one takes what fits the `room` left at the average size so far. +fn chunk_len(left: usize, room: Option, (bytes, entries): (usize, usize)) -> usize { + let Some(room) = room else { + return left; + }; + if entries == 0 { + return 1; + } + let average = (bytes / entries).max(1); + (room / average + 1).clamp(1, left.max(1)) +} + +fn decode_record(bytes: &[u8], offset: i64) -> StreamResult { + let mut record = StreamRecord::decode(bytes).map_err(|error| { + StreamError::storage_corrupt(format!("A stored Record is bad: {error}")) + })?; + record.offset = offset_to_wire(offset); + Ok(record) +} + +/// Fails a read whose Records the log lost or the expiry removed, so that the Reader always sees +/// the gap. A read from the start of the Stream has no gap. +/// +/// A Tombstone fails every read. An open Stream that expired gets no more Records, so the Reader +/// would wait forever. A closed one lost its log. +fn check_expiry(state: &StreamState, from: i64) -> StreamResult<()> { + if state.expired() || (from > 0 && state.trimmed >= from) { + return Err(expired()); + } + Ok(()) +} + +/// The byte cap of a page, from a request's `max_bytes`. Zero is no cap. +fn max_bytes(value: i64) -> StreamResult { + match value { + 0 => Ok(usize::MAX), + n if n < 0 => Err(StreamError::invalid("max_bytes must be 0 or more")), + n => Ok(usize::try_from(n).unwrap_or(usize::MAX)), + } +} + +fn expired() -> StreamError { + StreamError::new(Reason::Expired, "The Records at the offset expired") +} diff --git a/crates/streams/src/record.rs b/crates/streams/src/record.rs new file mode 100644 index 000000000..ca0607a74 --- /dev/null +++ b/crates/streams/src/record.rs @@ -0,0 +1,118 @@ +//! The stored Records of a batch. + +use crate::{ + Reason, StreamError, StreamResult, + proto::{AppendRecord, Producer, StreamRecord}, + store::{MAX_BATCH_BYTES, MAX_BATCH_RECORDS, MAX_RECORD_BYTES}, +}; +use prost::Message; + +/// One stored data Record. It has no offset, since the Store gives the entry its offset. +pub(crate) fn data_record(producer: &Producer, record: AppendRecord) -> Vec { + StreamRecord { + producer: Some(producer.clone()), + body: record.body, + ..Default::default() + } + .encode_to_vec() +} + +/// Refuses a batch with a Record that has no body, before the Store. +pub(crate) fn check_records(records: &[AppendRecord]) -> StreamResult<()> { + match records.iter().position(|record| record.body.is_none()) { + Some(at) => Err(StreamError::invalid(format!( + "Record {at} of the batch has no body" + ))), + None => Ok(()), + } +} + +/// The stored Records of one batch from outside Workflow code. A batch over the Store's limits +/// fails, since the crate doesn't split a batch it didn't build. +pub(crate) fn data_records( + producer: &Producer, + records: Vec, +) -> StreamResult>> { + if records.len() > MAX_BATCH_RECORDS { + return Err(StreamError::new( + Reason::TooLarge, + format!( + "The batch has {} Records, more than {MAX_BATCH_RECORDS}", + records.len() + ), + )); + } + let records: Vec<_> = records + .into_iter() + .map(|record| data_record(producer, record)) + .collect(); + if let Some(big) = records + .iter() + .find(|record| record.len() > MAX_RECORD_BYTES) + { + return Err(StreamError::new( + Reason::TooLarge, + format!( + "A Record has {} bytes, more than {MAX_RECORD_BYTES}", + big.len() + ), + )); + } + let total: usize = records.iter().map(Vec::len).sum(); + if total > MAX_BATCH_BYTES { + return Err(StreamError::new( + Reason::TooLarge, + format!("The batch has {total} bytes, more than {MAX_BATCH_BYTES}"), + )); + } + Ok(records) +} + +#[cfg(test)] +mod tests { + use super::*; + use temporalio_common::protos::temporal::api::common::v1::Payload; + + #[test] + fn a_record_with_no_body_is_invalid() { + let body = AppendRecord { + body: Some(Payload::default()), + }; + check_records(std::slice::from_ref(&body)).unwrap(); + let error = check_records(&[body, AppendRecord::default()]).unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + assert!(error.message.contains("Record 1"), "{}", error.message); + } + + #[test] + fn a_batch_over_the_limits_fails() { + let producer = Producer::default(); + let record = |bytes: usize| AppendRecord { + body: Some(Payload { + data: vec![0; bytes], + ..Default::default() + }), + }; + let error = data_records(&producer, vec![record(MAX_RECORD_BYTES)]).unwrap_err(); + assert_eq!(error.reason, Reason::TooLarge); + let many = (0..4) + .map(|_| record(MAX_RECORD_BYTES / 2 + 1)) + .collect::>(); + let many = [many.clone(), many].concat(); + assert_eq!( + data_records(&producer, many).unwrap_err().reason, + Reason::TooLarge + ); + assert_eq!(data_records(&producer, vec![record(10)]).unwrap().len(), 1); + let small = vec![record(1); MAX_BATCH_RECORDS + 1]; + assert_eq!( + data_records(&producer, small).unwrap_err().reason, + Reason::TooLarge + ); + let small = vec![record(1); MAX_BATCH_RECORDS]; + assert_eq!( + data_records(&producer, small).unwrap().len(), + MAX_BATCH_RECORDS + ); + } +} diff --git a/crates/streams/src/runtime.rs b/crates/streams/src/runtime.rs new file mode 100644 index 000000000..dc590d911 --- /dev/null +++ b/crates/streams/src/runtime.rs @@ -0,0 +1,268 @@ +//! The Stream runtime, a plain object that lang's bridge builds per client. + +use crate::{ + Reason, StreamError, StreamResult, + digest::batch_digest, + epoch::ProducerKind, + ids::{check_producer, check_required, check_stream_id}, + memory::MemoryStore, + metrics::Metrics, + offset::offset_to_wire, + proto::{ + AppendRecordsRequest, AppendRecordsResponse, CloseStreamRequest, CloseStreamResponse, + CreateStreamRequest, CreateStreamResponse, DescribeStreamRequest, DescribeStreamResponse, + Producer, StreamIdConflictPolicy, StreamRef, StreamStatus, + }, + record::{check_records, data_records}, + store::{ + Batch, Capabilities, ConflictPolicy, Create, CreateOutcome, DEFAULT_RETENTION, + MAX_SEQUENCE, Store, + }, +}; +use std::{sync::Arc, time::Duration}; +use temporalio_common::telemetry::metrics::TemporalMeter; + +/// The Stream runtime on one Store. It serves the five Stream service operations. A Store has no +/// namespaces, so the runtime ignores a request's `namespace`. The SDK sets it, and only the +/// server reads it. +/// +/// A bridge builds one for each client with a Stream Store, and that client's Workers share it. +/// A Store crate, such as `temporalio-streams-redis`, builds it with its own Store. +pub struct StreamRuntime { + /// The name of the Store, which the refs that the runtime builds carry. + store_name: String, + pub(crate) store: Arc, + pub(crate) metrics: Metrics, +} + +impl std::fmt::Debug for StreamRuntime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StreamRuntime") + .field("store_name", &self.store_name) + .finish_non_exhaustive() + } +} + +pub(crate) fn duration(value: Option) -> Option { + let value = value?; + let duration = Duration::new(value.seconds.max(0) as u64, value.nanos.max(0) as u32); + (!duration.is_zero()).then_some(duration) +} + +/// The conflict policy of a create. Unspecified is FAIL, as for a Workflow start. +fn conflict_policy(value: i32) -> StreamResult { + match StreamIdConflictPolicy::try_from(value) { + Ok(StreamIdConflictPolicy::Unspecified | StreamIdConflictPolicy::Fail) => { + Ok(ConflictPolicy::Fail) + } + Ok(StreamIdConflictPolicy::UseExisting) => Ok(ConflictPolicy::UseExisting), + Err(_) => Err(StreamError::invalid(format!( + "The conflict policy {value} is unknown" + ))), + } +} + +pub(crate) fn stream_id(stream_ref: Option<&StreamRef>) -> StreamResult<&str> { + let id = stream_ref + .map(|stream_ref| stream_ref.id.as_str()) + .unwrap_or_default(); + check_stream_id(id)?; + Ok(id) +} + +/// Checks the Producer of a close. Any kind may close. +fn close_producer(producer: Option<&Producer>) -> StreamResult<&Producer> { + let producer = producer.ok_or_else(|| StreamError::invalid("The request has no Producer"))?; + check_producer(producer)?; + check_required(producer)?; + Ok(producer) +} + +impl StreamRuntime { + /// Builds a runtime on `store`. `store_name` is the name that lang registered the Store under, + /// which the refs that the runtime builds carry. The runtime records metrics on `meter`, lang's + /// runtime meter, if set. A Store that connects later, such as a [crate::store::LazyStore], + /// lets a bridge build the runtime with no async runtime. + pub fn new( + store_name: impl Into, + store: Arc, + meter: Option, + ) -> Arc { + let meter = meter.unwrap_or_else(TemporalMeter::no_op); + Arc::new(Self { + store_name: store_name.into(), + store, + metrics: Metrics::new(&meter), + }) + } + + /// Creates a Stream, or gets it with USE_EXISTING. + pub async fn create_stream( + &self, + request: CreateStreamRequest, + ) -> StreamResult { + stream_id(request.stream_ref.as_ref())?; + let stream = request.stream_ref.clone().unwrap_or_default(); + let conflict = conflict_policy(request.id_conflict_policy)?; + // The Store config holds the Retention, so a create takes the Store default. + let create = Create { + retention: None, + conflict, + request_id: &request.request_id, + }; + let created = self.store.create(&stream, &create).await?; + Ok(CreateStreamResponse { + r#ref: Some(StreamRef { + id: stream.id, + store: self.store_name.clone(), + }), + created: created.outcome == CreateOutcome::Created, + }) + } + + /// Appends one batch as one Producer. A Workflow appends as the Producer of its Local Activity, + /// so a Workflow Producer is refused. + pub async fn append_records( + &self, + request: AppendRecordsRequest, + ) -> StreamResult { + stream_id(request.r#ref.as_ref())?; + let stream = request.r#ref.clone().unwrap_or_default(); + let producer = request + .producer + .as_ref() + .ok_or_else(|| StreamError::invalid("The request has no Producer"))?; + check_producer(producer)?; + if ProducerKind::try_from(producer)? == ProducerKind::Workflow { + return Err(StreamError::invalid( + "A Workflow writes with the Producer of its Local Activity", + )); + } + check_required(producer)?; + if !(0..=MAX_SEQUENCE).contains(&request.sequence) { + return Err(StreamError::invalid( + "The sequence must be from 1 to 2^53 - 1, or zero for the next one", + )); + } + check_records(&request.records)?; + // Without the fence, a stale batch would reach Readers. + let capabilities = self.store.capabilities(); + capabilities.require(Capabilities::PRODUCER_FENCE)?; + if request.sequence == 0 { + // Only the Store can number the batch in the same step as its dedupe check. + capabilities.require(Capabilities::NEXT_SEQUENCE)?; + } + let digest = batch_digest(&request.records); + let records = data_records(producer, request.records)?; + let batch = Batch { + producer, + sequence: request.sequence, + records: &records, + digest, + }; + let appended = self.store.append(&stream, &batch).await; + if let Err(error) = &appended + && error.reason == Reason::Fenced + { + self.metrics.fenced(); + } + // An empty batch has no offset, which is the empty wire value. + let (first_offset, last_offset) = appended?.offsets.unwrap_or_default(); + Ok(AppendRecordsResponse { + first_offset: offset_to_wire(first_offset), + last_offset: offset_to_wire(last_offset), + }) + } + + /// Closes a Stream. Its readers drain what is left, and later appends fail. + pub async fn close_stream( + &self, + request: CloseStreamRequest, + ) -> StreamResult { + stream_id(request.r#ref.as_ref())?; + let stream = request.r#ref.clone().unwrap_or_default(); + let producer = close_producer(request.producer.as_ref())?; + self.store.capabilities().require(Capabilities::CLOSE)?; + self.store.close(&stream, producer).await?; + Ok(CloseStreamResponse {}) + } + + /// Gets the status and the newest offset of a Stream. + pub async fn describe_stream( + &self, + request: DescribeStreamRequest, + ) -> StreamResult { + stream_id(request.r#ref.as_ref())?; + let stream = request.r#ref.clone().unwrap_or_default(); + let (state, _) = self.store.read(&stream, 0, 0).await?; + if !state.exists { + return Err(StreamError::new( + Reason::NotFound, + "Nobody created the Stream", + )); + } + // A closed Stream that expired has lost its log, and every read fails. + let status = if state.expired() { + StreamStatus::Expired + } else if state.closed { + StreamStatus::Closed + } else { + StreamStatus::Open + }; + Ok(DescribeStreamResponse { + status: status as i32, + last_offset: offset_to_wire(state.newest), + }) + } +} + +/// The settings of the memory Store runtime. A field left `None` takes its default. +#[derive(Clone, Debug, Default)] +pub struct MemoryStreamOptions { + /// The Retention of each Stream. `None` is [DEFAULT_RETENTION]. + pub retention: Option, + /// How long an open Stream stays after its last write. `None` keeps an open Stream until its + /// close. + pub last_write_ttl: Option, +} + +/// The Stream runtime on the memory Store, for tests. It serves the same calls with the same +/// rules as every other Store, and it keeps its Streams in this process only. +#[derive(Debug)] +pub struct MemoryStreamRuntime; + +impl MemoryStreamRuntime { + /// Builds a runtime on a new memory Store. A zero duration in `options` is the same as `None`. + /// The type only names this constructor, so it returns the runtime and not `Self`. + #[allow(clippy::new_ret_no_self)] + pub fn new( + store_name: impl Into, + options: MemoryStreamOptions, + meter: Option, + ) -> Arc { + let positive = |value: Option| value.filter(|value| !value.is_zero()); + let store = Arc::new(MemoryStore::new( + positive(options.retention).unwrap_or(DEFAULT_RETENTION), + positive(options.last_write_ttl), + )); + StreamRuntime::new(store_name, store, meter) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_zero_or_negative_duration_is_unset_and_a_huge_one_stays() { + let proto = |seconds, nanos| Some(prost_types::Duration { seconds, nanos }); + assert_eq!(duration(None), None); + assert_eq!(duration(proto(0, 0)), None); + assert_eq!(duration(proto(-5, 0)), None); + assert_eq!(duration(proto(0, -5)), None); + assert_eq!( + duration(proto(i64::MAX, 999_999_999)), + Some(Duration::new(i64::MAX as u64, 999_999_999)) + ); + } +} diff --git a/crates/streams/src/store.rs b/crates/streams/src/store.rs new file mode 100644 index 000000000..3b9638428 --- /dev/null +++ b/crates/streams/src/store.rs @@ -0,0 +1,285 @@ +//! The calls that every Store gives the Stream service. The Redis Store of +//! `temporalio-streams-redis` is for production, and the memory Store is for tests. Both apply the +//! same rules. + +use crate::{ + Reason, StreamError, StreamResult, + proto::{Producer, StreamRef}, +}; +use std::time::Duration; +use tokio::time::Instant; + +mod lazy; + +pub use lazy::LazyStore; + +/// The Retention of a Stream whose create sets none, when a Store's options set no other. +pub const DEFAULT_RETENTION: Duration = Duration::from_secs(7 * 24 * 60 * 60); + +/// The format version this crate writes. A Stream with a higher one refuses writes. +pub const FORMAT_VERSION: u32 = 1; +/// The maximum size of a stored Record, in bytes. +pub const MAX_RECORD_BYTES: usize = 1536 * 1024; +/// The maximum size of a stored append, in bytes. +pub const MAX_BATCH_BYTES: usize = 4 * 1024 * 1024; +/// The maximum number of Records in an append. Many small Records fit under [MAX_BATCH_BYTES], but +/// they keep a Store busy for long. +pub const MAX_BATCH_RECORDS: usize = 1000; +/// The highest batch `sequence`. A Lua number is exact for integers only up to this value. +pub const MAX_SEQUENCE: i64 = (1 << 53) - 1; +/// How long the meta key stays after the log key expires, so that a read can tell an expired Stream +/// from a missing one. +pub const TOMBSTONE: Duration = Duration::from_secs(30 * 24 * 60 * 60); +/// The longest Retention or last-write TTL, which is the largest proto Duration. Its ms plus the +/// time of a write stay exact in a Lua number. +pub const MAX_TTL: Duration = Duration::from_secs(315_576_000_000); + +/// What a create does when its Stream exists and is open, and the create isn't a retry of the +/// create that made the Stream. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum ConflictPolicy { + /// The create fails with [Reason::AlreadyExists]. + #[default] + Fail, + /// The create gets the open Stream, which keeps its own Retention. + UseExisting, +} + +/// A create of a Stream. +#[derive(Debug, Clone, Default)] +pub struct Create<'a> { + /// The Retention, `None` for the Store default. The service always sends `None`, since the + /// Store config holds the Retention. A test may set a short one. + pub retention: Option, + /// What the create does when the Stream exists. + pub conflict: ConflictPolicy, + /// Unique for each create call, and the same on its retries. Empty for none. The Store keeps + /// the request id of the create that made the Stream, so a retry after a lost answer succeeds. + pub request_id: &'a str, +} + +/// What a create found. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CreateOutcome { + /// The create wrote the key, or it is a retry of the create that wrote it. + Created, + /// The Stream exists and is open, and the create uses it. The create doesn't change its + /// Retention. + Exists, +} + +/// The result of a create. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Created { + /// What the create found. + pub outcome: CreateOutcome, + /// The Retention of the Stream, the stored one if it existed. + pub retention: Duration, +} + +/// One batch of one Producer. +#[derive(Debug, Clone)] +pub struct Batch<'a> { + /// The Producer, with its epoch. + pub producer: &'a Producer, + /// The batch number in this epoch. Zero takes the next one of the epoch, which needs + /// [Capabilities::NEXT_SEQUENCE]. Such a batch has no dedupe, so its retry writes it again. + pub sequence: i64, + /// The encoded Records. + pub records: &'a [Vec], + /// The [crate::digest::batch_digest] of the batch, which the dedupe entry keeps. + pub digest: [u8; crate::digest::DIGEST_LEN], +} + +/// The result of an append. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Appended { + /// The offsets of the first and the last entry, `None` for an empty batch. + pub offsets: Option<(i64, i64)>, + /// The Store already had the batch, so the offsets are those of the first result. + pub resend: bool, +} + +/// The result of a close. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Closed { + /// The Stream was already closed. + pub already: bool, +} + +/// The state of a Stream as one read saw it. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct StreamState { + /// The meta key exists, live or as a Tombstone. + pub exists: bool, + /// The format version. + pub version: u32, + /// The Stream is closed. Every later append fails, so `newest` is final. + pub closed: bool, + /// The offset of the newest entry written, zero if none. + pub newest: i64, + /// The newest offset when a write found the log gone, zero if none. + pub trimmed: i64, + /// When the Stream stops taking writes, in ms since 1970. Zero for an open Stream with no + /// last-write TTL, which stays until its close. + pub expires_ms: i64, + /// The server time of the read, in ms since 1970. + pub now_ms: i64, +} + +impl StreamState { + /// The Stream is a Tombstone. + pub fn expired(&self) -> bool { + self.exists && self.expires_ms > 0 && self.expires_ms <= self.now_ms + } + + /// The offset below which the log lost the Records. The Retention counts from the close, so + /// it hides no Record of a Stream that hasn't expired. + pub fn retained_from(&self) -> i64 { + self.trimmed + 1 + } +} + +/// One log entry. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Entry { + /// Its offset. + pub offset: i64, + /// The encoded Record. + pub record: Option>, +} + +/// The rules that a Store applies atomically beyond the log. The service checks a capability before +/// a call that needs it. So a Store that lacks the capability refuses the call, and doesn't give +/// weaker guarantees without notice. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Capabilities(u8); + +impl Capabilities { + /// Each append checks the Producer's epoch and its last `sequence` before it writes. The Store + /// keeps both for as long as the Stream exists, also for a Producer with no recent append. + pub const PRODUCER_FENCE: Self = Self(1); + /// A close marks the Stream closed and wakes the waiting reads. Every later append is + /// refused. + pub const CLOSE: Self = Self(1 << 1); + /// An append with sequence zero takes the next one of the Producer's epoch, in the same atomic + /// step as the dedupe check. So two such batches never get the same number. + pub const NEXT_SEQUENCE: Self = Self(1 << 2); + /// Every capability. + pub const ALL: Self = Self((1 << 3) - 1); + + const NAMES: [(Self, &'static str); 3] = [ + (Self::PRODUCER_FENCE, "producer_fence"), + (Self::CLOSE, "close"), + (Self::NEXT_SEQUENCE, "next_sequence"), + ]; + + /// Both sets. + pub const fn union(self, other: Self) -> Self { + Self(self.0 | other.0) + } + + /// This set without the other one. + pub const fn without(self, other: Self) -> Self { + Self(self.0 & !other.0) + } + + /// Whether every capability of `other` is in this set. + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } + + /// The names of the capabilities in this set. + pub fn names(self) -> Vec<&'static str> { + Self::NAMES + .iter() + .filter(|(capability, _)| self.contains(*capability)) + .map(|(_, name)| *name) + .collect() + } + + /// Refuses a call that needs a capability this set lacks, and names the missing ones. + pub fn require(self, needed: Self) -> StreamResult<()> { + let missing = needed.without(self); + if missing == Self::default() { + return Ok(()); + } + Err(unsupported(&missing.names().join(", "))) + } +} + +fn unsupported(capability: &str) -> StreamError { + StreamError::new( + Reason::Unsupported, + format!("The Store lacks the capability {capability}"), + ) +} + +/// The calls of a Store. Each one is atomic, as one Lua script is on Redis. +/// +/// The required calls make the log contract. They are a create key, an atomic batch append, an +/// ordered read with the Stream's state, stable offsets, a Retention that counts from the close and +/// a wait for the next entry. An append applies the rules of the capabilities that the Store +/// declares. The optional calls fail by default. +#[async_trait::async_trait] +pub trait Store: Send + Sync + 'static { + /// What the Store does atomically beyond the log. It has no default, so a Store that wraps + /// another one must say what it forwards. + fn capabilities(&self) -> Capabilities; + + /// Creates a Stream. + /// + /// The create of an existing Stream fails with [Reason::Expired] or [Reason::Closed], in this + /// order. Then a retry of the create that made the Stream, with the same request id, gets + /// [CreateOutcome::Created] and changes nothing. Any other create applies its [ConflictPolicy]. + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult; + + /// Appends the whole batch, or nothing of it. The batch's Records get consecutive offsets that + /// end at the last offset of [Appended::offsets], so a caller knows the offset of each one. + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult; + + /// Reads the Stream's state and up to `count` entries from `from`, inclusive. Zero reads + /// from the start of the log. + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)>; + + /// Waits until the log has an entry after `after`, the Stream is closed, or `deadline` passes. + /// Returns whether an entry came or the Stream is closed. It may return `false` before the + /// deadline, and then the caller reads again. + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult; + + /// Closes a Stream. A close writes no log entry, and a close of a closed Stream changes + /// nothing. Needs [Capabilities::CLOSE]. + async fn close(&self, _stream_ref: &StreamRef, _producer: &Producer) -> StreamResult { + Err(unsupported("close")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_missing_capability_is_refused_by_name() { + let some = Capabilities::PRODUCER_FENCE.union(Capabilities::CLOSE); + some.require(Capabilities::CLOSE).unwrap(); + let error = some + .require(Capabilities::CLOSE.union(Capabilities::NEXT_SEQUENCE)) + .unwrap_err(); + assert_eq!(error.reason, Reason::Unsupported); + assert!( + error.message.ends_with("next_sequence"), + "{}", + error.message + ); + } +} diff --git a/crates/streams/src/store/lazy.rs b/crates/streams/src/store/lazy.rs new file mode 100644 index 000000000..67771b7e4 --- /dev/null +++ b/crates/streams/src/store/lazy.rs @@ -0,0 +1,181 @@ +//! A Store that connects on its first call. + +use crate::{ + StreamError, StreamResult, + proto::{Producer, StreamRef}, + store::{Appended, Batch, Capabilities, Closed, Create, Created, Entry, Store, StreamState}, +}; +use std::{future::Future, pin::Pin}; +use tokio::{sync::OnceCell, time::Instant}; + +type Connect = + Box Pin> + Send>> + Send + Sync>; + +/// A Store that connects its inner Store on the first call, so a bridge can build the runtime with +/// no async runtime, and a Workflow that never publishes never connects. A failed connect is tried +/// again on the next call. An inner Store that lacks a declared capability is refused once, and +/// every later call fails with that refusal. +pub struct LazyStore { + capabilities: Capabilities, + connect: Connect, + store: OnceCell>, +} + +impl std::fmt::Debug for LazyStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LazyStore") + .field("capabilities", &self.capabilities) + .field("connected", &matches!(self.store.get(), Some(Ok(_)))) + .finish_non_exhaustive() + } +} + +impl LazyStore { + /// A Store that `connect` connects on its first call. The runtime asks for the capabilities + /// before any call connects, so the Store declares `capabilities` when it is built, and a + /// connect fails if the inner Store lacks one of them. + pub fn new(capabilities: Capabilities, connect: F) -> Self + where + F: Fn() -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + E: Into, + { + Self { + capabilities, + connect: Box::new(move || { + let connecting = connect(); + Box::pin(async move { connecting.await.map_err(Into::into) }) + }), + store: OnceCell::new(), + } + } + + async fn store(&self) -> StreamResult<&S> { + let connected = self + .store + .get_or_try_init(|| async { + let store = (self.connect)().await?; + // The same Store lacks the capability on the next connect too. + Ok::<_, StreamError>( + store + .capabilities() + .require(self.capabilities) + .map(|()| store), + ) + }) + .await?; + connected.as_ref().map_err(Clone::clone) + } +} + +#[async_trait::async_trait] +impl Store for LazyStore { + fn capabilities(&self) -> Capabilities { + self.capabilities + } + + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult { + self.store().await?.create(stream_ref, create).await + } + + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult { + self.store().await?.append(stream_ref, batch).await + } + + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)> { + self.store().await?.read(stream_ref, from, count).await + } + + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult { + self.store() + .await? + .wait_after(stream_ref, after, deadline) + .await + } + + async fn close(&self, stream_ref: &StreamRef, producer: &Producer) -> StreamResult { + self.store().await?.close(stream_ref, producer).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{Reason, memory::MemoryStore, store::DEFAULT_RETENTION}; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + fn stream() -> StreamRef { + StreamRef { + id: "s".to_string(), + ..Default::default() + } + } + + fn memory() -> MemoryStore { + MemoryStore::new(DEFAULT_RETENTION, None) + } + + #[tokio::test] + async fn a_failed_connect_is_tried_again_on_the_next_call() { + let connects = Arc::new(AtomicUsize::new(0)); + let counted = connects.clone(); + let lazy = LazyStore::new(Capabilities::ALL, move || { + let first = counted.fetch_add(1, Ordering::SeqCst) == 0; + async move { + if first { + return Err(StreamError::storage_unavailable("the Store is down")); + } + Ok(memory()) + } + }); + assert_eq!(lazy.capabilities(), Capabilities::ALL); + assert_eq!(connects.load(Ordering::SeqCst), 0); + let error = lazy + .create(&stream(), &Create::default()) + .await + .unwrap_err(); + assert_eq!(error.reason, Reason::StorageUnavailable); + lazy.create(&stream(), &Create::default()).await.unwrap(); + let (state, _) = lazy.read(&stream(), 0, 0).await.unwrap(); + assert!(state.exists); + assert_eq!(connects.load(Ordering::SeqCst), 2); + } + + #[tokio::test] + async fn a_connect_refuses_a_store_that_lacks_a_declared_capability_once() { + let connects = Arc::new(AtomicUsize::new(0)); + let counted = connects.clone(); + let lazy = LazyStore::new(Capabilities::ALL, move || { + counted.fetch_add(1, Ordering::SeqCst); + async { + let fewer = LazyStore::new(Capabilities::CLOSE, || async { + Ok::<_, StreamError>(memory()) + }); + Ok::<_, StreamError>(fewer) + } + }); + for _ in 0..2 { + let error = lazy.read(&stream(), 0, 0).await.unwrap_err(); + assert_eq!(error.reason, Reason::Unsupported); + assert!( + error.message.contains("producer_fence"), + "{}", + error.message + ); + } + assert_eq!(connects.load(Ordering::SeqCst), 1); + } +} diff --git a/crates/streams/src/testing/fields.rs b/crates/streams/src/testing/fields.rs new file mode 100644 index 000000000..75fd320ea --- /dev/null +++ b/crates/streams/src/testing/fields.rs @@ -0,0 +1,32 @@ +//! The fields of the `:meta` hash. A Producer field is `p||`. The id is opaque +//! and ends the name, so a `|` in it makes no other field. + +/// The format version of the Stream. +pub const VERSION: &str = "v"; + +/// The field of a Producer. Its value is 88 bytes, with big-endian numbers: +/// +/// - 40, the fence, the Producer's newest epoch key. It is also the epoch of the last batch. +/// - 8, the last batch's sequence. +/// - 12 and 12, the last batch's first and last entry ids, each the ms in 8 bytes and the seq in +/// 4. They are zero when the batch wrote no entry. +/// - 16, the last batch's digest, which a resend must match. +/// +/// A Store refuses a value of another length. +pub fn producer(kind: i32, producer_id: &str) -> String { + format!("p|{kind}|{producer_id}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_fields_match_the_shared_vectors() { + let case: serde_json::Value = + serde_json::from_str(include_str!("../../testdata/fields.json")).unwrap(); + let text = |field: &str| case[field].as_str().unwrap(); + let kind = case["kind"].as_i64().unwrap() as i32; + assert_eq!(producer(kind, text("producer_id")), text("producer")); + } +} diff --git a/crates/streams/src/testing/mod.rs b/crates/streams/src/testing/mod.rs new file mode 100644 index 000000000..0879aeffe --- /dev/null +++ b/crates/streams/src/testing/mod.rs @@ -0,0 +1,170 @@ +//! The shared suites of the Stream rules. Each Store crate runs them on its own Stores, so that +//! every Store keeps the same rules. This crate runs them on the memory Store. + +mod support; + +pub mod fields; +pub mod service; +pub mod store; + +pub use support::Needs; + +use crate::{ + MemoryStreamOptions, MemoryStreamRuntime, StreamRuntime, + memory::MemoryStore, + store::{DEFAULT_RETENTION, Store}, +}; +use futures_util::{FutureExt, future::join_all}; +use std::{future::Future, panic::AssertUnwindSafe, pin::Pin, sync::Arc, time::Duration}; + +/// The state under a Store's calls, which some cases read or change. +#[async_trait::async_trait] +pub trait RawKeys: Send + Sync { + /// Sets one meta field of a Stream. Field names and values are bytes, as Redis keeps them. + async fn set_meta(&self, stream: &str, field: &[u8], value: &[u8]); + /// One meta field of a Stream. + async fn meta(&self, stream: &str, field: &[u8]) -> Option>; + /// The ms left on the log and on the meta, as `PTTL` gives them. + async fn ttls(&self, stream: &str) -> (i64, i64); + /// Adds a log entry with an explicit id, as `XADD `. + async fn add_raw_entry(&self, stream: &str, id: &str); + /// The number of log entries. + async fn log_len(&self, stream: &str) -> usize; + /// Deletes the log, as an eviction would. + async fn drop_log(&self, stream: &str); +} + +/// The settings of a runtime under test. A field left `None` keeps the Store's own value. +#[derive(Clone, Debug, Default)] +pub struct RuntimeConfig { + /// The Store name that the runtime sets on each ref it builds. + pub store_name: String, + /// The Retention of each Stream that the runtime creates. + pub retention: Option, + /// How long an open Stream stays after its last write. + pub last_write_ttl: Option, +} + +/// A Store that the suites run on, with the server under it. +#[async_trait::async_trait] +pub trait TestStore: Send + Sync { + /// What the Store gives. A case that needs more doesn't run on it. + fn gives(&self) -> Needs; + /// A Store for one case, and the state under it. + async fn connect(&self) -> (Arc, Arc); + /// A runtime on the Store with `config`. + fn runtime(&self, config: &RuntimeConfig) -> Arc; +} + +/// The memory Store, with a new Store for each case. +#[derive(Debug, Default)] +pub struct MemoryTestStore; + +#[async_trait::async_trait] +impl TestStore for MemoryTestStore { + fn gives(&self) -> Needs { + Needs::ALL + } + + async fn connect(&self) -> (Arc, Arc) { + let store = Arc::new(MemoryStore::new(DEFAULT_RETENTION, None)); + (store.clone(), store) + } + + fn runtime(&self, config: &RuntimeConfig) -> Arc { + let options = MemoryStreamOptions { + retention: config.retention, + last_write_ttl: config.last_write_ttl, + }; + MemoryStreamRuntime::new(config.store_name.clone(), options, None) + } +} + +#[async_trait::async_trait] +impl RawKeys for MemoryStore { + async fn set_meta(&self, stream: &str, field: &[u8], value: &[u8]) { + self.test_set_meta(stream, field, value) + } + + async fn meta(&self, stream: &str, field: &[u8]) -> Option> { + self.test_meta(stream, field) + } + + async fn ttls(&self, stream: &str) -> (i64, i64) { + self.test_ttls(stream) + } + + async fn add_raw_entry(&self, stream: &str, id: &str) { + self.test_add_entry(stream, id) + } + + async fn log_len(&self, stream: &str) -> usize { + self.test_log_len(stream) + } + + async fn drop_log(&self, stream: &str) { + self.test_drop_log(stream) + } +} + +/// One case of a suite. +pub struct Case { + /// The name of the case, for the report. + pub name: &'static str, + /// Runs the case on each Store under test that gives what it needs. + pub run: fn() -> Pin>>, +} + +macro_rules! case { + ($name:ident) => { + $crate::testing::Case { + name: stringify!($name), + run: || Box::pin($name()), + } + }; +} +pub(crate) use case; + +tokio::task_local! { + static TEST_STORES: Vec>; +} + +/// The Stores of the running suite. +pub(crate) fn test_stores() -> Vec> { + TEST_STORES.with(Clone::clone) +} + +/// Runs `cases` on `stores`, all at once, and then fails with the names of the cases that +/// failed. A case that fails prints its panic as it goes. +pub async fn run(stores: Vec>, cases: &[Case]) { + let outcomes = TEST_STORES + .scope( + stores, + join_all(cases.iter().map(|case| async move { + let passed = AssertUnwindSafe((case.run)()).catch_unwind().await.is_ok(); + (case.name, passed) + })), + ) + .await; + let failed: Vec<_> = outcomes + .into_iter() + .filter(|(_, passed)| !passed) + .map(|(name, _)| name) + .collect(); + assert!(failed.is_empty(), "these cases failed: {failed:?}"); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn the_store_suite_passes_on_the_memory_store() { + run(vec![Arc::new(MemoryTestStore)], store::CASES).await; + } + + #[tokio::test] + async fn the_service_suite_passes_on_the_memory_store() { + run(vec![Arc::new(MemoryTestStore)], service::CASES).await; + } +} diff --git a/crates/streams/src/testing/service.rs b/crates/streams/src/testing/service.rs new file mode 100644 index 000000000..4fa5a3b08 --- /dev/null +++ b/crates/streams/src/testing/service.rs @@ -0,0 +1,974 @@ +//! The Stream service operations through the runtime's typed calls. +//! +//! Every Store runs the suite through its own runtime. + +use super::{RuntimeConfig, case, support::Needs}; + +use crate::{ + StreamError, StreamRuntime, + ids::MAX_PRODUCER_ID_BYTES, + proto::{ + ActivityEpoch, ActivityProducer, AppendRecord, AppendRecordsRequest, AppendRecordsResponse, + ClientProducer, CloseStreamRequest, CloseStreamResponse, CreateStreamRequest, + CreateStreamResponse, DescribeStreamRequest, DescribeStreamResponse, LocalActivityEpoch, + LocalActivityProducer, Producer, ReadRecordsRequest, ReadRecordsResponse, + StreamErrorDetails, StreamIdConflictPolicy, StreamRef, StreamStatus, WorkflowProducer, + WorkflowTaskEpoch, producer::Kind, + }, +}; +use prost::Message; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; +use temporalio_common::protos::temporal::api::common::v1::Payload; +use tonic::Code; + +struct Case { + ext: Arc, + namespace: String, + stream: String, +} + +static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + +fn unique() -> String { + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + format!("{now}-{n}") +} + +/// The Retention of the cases that let a closed Stream expire. +const SHORT_RETENTION: Duration = Duration::from_millis(300); + +/// A fresh Stream on each Store that gives `needs`. +async fn cases_for(needs: Needs) -> Vec { + cases_with(needs, &RuntimeConfig::default()).await +} + +/// A fresh Stream on each Store that gives short times, on a runtime whose Retention is +/// [SHORT_RETENTION]. +async fn short_cases() -> Vec { + let config = RuntimeConfig { + retention: Some(SHORT_RETENTION), + ..Default::default() + }; + cases_with(Needs::SHORT_TIMES, &config).await +} + +/// A fresh Stream on each Store that gives `needs`, on a runtime with `config`. +async fn cases_with(needs: Needs, config: &RuntimeConfig) -> Vec { + super::support::runtimes(needs, config) + .await + .into_iter() + .map(|runtime| { + let unique = unique(); + Case { + ext: runtime.ext, + namespace: runtime.namespace.unwrap_or(format!("ns-{unique}")), + stream: format!("s-{unique}"), + } + }) + .collect() +} + +fn reason(error: &StreamError) -> String { + let details = StreamErrorDetails::decode(error.details().as_slice()).unwrap(); + assert_eq!(details.reason(), error.reason.proto()); + error.reason.token().to_string() +} + +/// The internal offset of a wire value, for the suite's assertions. +fn off(wire: &[u8]) -> i64 { + crate::offset::offset_from_wire(wire).unwrap() +} + +/// The wire value of an internal offset. +fn wire(offset: i64) -> Vec { + crate::offset::offset_to_wire(offset) +} + +fn client() -> Producer { + Producer { + id: "client:me:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } +} + +/// The Producer of run `run` in its first Workflow Task, as its close Activity sends it. +fn workflow(run: &str) -> Producer { + Producer { + id: format!("wf:{run}"), + kind: Some(Kind::Workflow(WorkflowProducer { + run_id: run.to_string(), + epoch: Some(WorkflowTaskEpoch { + started_event_id: 3, + attempt: 1, + started_time: None, + }), + })), + } +} + +/// The fields of a create that uses an existing Stream. +fn use_existing() -> CreateStreamRequest { + CreateStreamRequest { + id_conflict_policy: StreamIdConflictPolicy::UseExisting as i32, + ..Default::default() + } +} + +fn record(text: &str) -> AppendRecord { + AppendRecord { + body: Some(Payload { + data: text.as_bytes().to_vec(), + ..Default::default() + }), + } +} + +fn millis(ms: u64) -> Option { + let duration = Duration::from_millis(ms); + Some(prost_types::Duration { + seconds: duration.as_secs() as i64, + nanos: duration.subsec_nanos() as i32, + }) +} + +impl Case { + fn stream_ref(&self) -> Option { + Some(StreamRef { + id: self.stream.clone(), + ..Default::default() + }) + } + + async fn create(&self) -> Result { + self.create_with(CreateStreamRequest::default()).await + } + + /// Creates the Stream with the fields of `request`, in the case's namespace and ref. + async fn create_with( + &self, + request: CreateStreamRequest, + ) -> Result { + let request = CreateStreamRequest { + namespace: self.namespace.clone(), + stream_ref: self.stream_ref(), + ..request + }; + self.ext.create_stream(request).await + } + + async fn append( + &self, + producer: Producer, + sequence: i64, + records: Vec, + ) -> Result { + self.ext + .append_records(AppendRecordsRequest { + namespace: self.namespace.clone(), + r#ref: self.stream_ref(), + producer: Some(producer), + sequence, + records, + }) + .await + } + + async fn read(&self, request: ReadRecordsRequest) -> Result { + self.ext + .read_records(ReadRecordsRequest { + namespace: self.namespace.clone(), + r#ref: self.stream_ref(), + ..request + }) + .await + } + + async fn describe(&self) -> Result { + self.ext + .describe_stream(DescribeStreamRequest { + namespace: self.namespace.clone(), + r#ref: self.stream_ref(), + }) + .await + } + + async fn close(&self, producer: Producer) -> Result { + self.ext + .close_stream(CloseStreamRequest { + namespace: self.namespace.clone(), + r#ref: self.stream_ref(), + producer: Some(producer), + }) + .await + } +} + +fn texts(response: &ReadRecordsResponse) -> Vec { + response + .records + .iter() + .map(|record| String::from_utf8(record.body.clone().unwrap().data).unwrap()) + .collect() +} + +/// A create of an open Stream fails as `already_exists`, unless it uses the Stream or it is a +/// retry of the create that made the Stream. Such a retry has the same request id, and it gets +/// `created` true with either policy. A create needs a ref with an id. +async fn a_create_of_an_open_stream_fails_unless_it_uses_it_or_retries() { + for case in cases_for(Needs::CALLS).await { + let first = CreateStreamRequest { + request_id: "create-1".to_string(), + ..Default::default() + }; + assert!(case.create_with(first.clone()).await.unwrap().created); + let other = CreateStreamRequest { + request_id: "create-2".to_string(), + ..Default::default() + }; + for request in [other.clone(), CreateStreamRequest::default()] { + let error = case.create_with(request).await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::AlreadyExists, "stream_already_exists") + ); + } + let used = CreateStreamRequest { + request_id: other.request_id, + ..use_existing() + }; + assert!(!case.create_with(used).await.unwrap().created); + let retry = CreateStreamRequest { + request_id: first.request_id.clone(), + ..use_existing() + }; + for request in [first, retry] { + assert!(case.create_with(request).await.unwrap().created); + } + for stream_ref in [None, Some(StreamRef::default())] { + let request = CreateStreamRequest { + namespace: case.namespace.clone(), + stream_ref, + ..Default::default() + }; + let error = case.ext.create_stream(request).await.unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + } + } +} + +/// A create of a closed Stream fails as closed, before its conflict policy and before a retry of +/// the create that made the Stream. +async fn a_create_of_a_closed_stream_fails() { + for case in cases_for(Needs::CALLS).await { + let first = CreateStreamRequest { + request_id: "create-1".to_string(), + ..Default::default() + }; + case.create_with(first.clone()).await.unwrap(); + case.close(client()).await.unwrap(); + for request in [first, CreateStreamRequest::default()] { + let error = case.create_with(request).await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::FailedPrecondition, "stream_closed") + ); + } + } +} + +/// A close writes no Record, so the describe's last offset stays at the newest Record. A second +/// close of the closed Stream succeeds, and a later append fails as closed. +async fn a_close_writes_no_record_and_a_second_close_succeeds() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let written = case.append(client(), 1, vec![record("a")]).await.unwrap(); + case.close(client()).await.unwrap(); + case.close(client()).await.unwrap(); + let error = case + .append(client(), 2, vec![record("b")]) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_closed"); + let describe = DescribeStreamRequest { + namespace: case.namespace.clone(), + r#ref: case.stream_ref(), + }; + let described = case.ext.describe_stream(describe).await.unwrap(); + assert_eq!(described.status, StreamStatus::Closed as i32); + assert_eq!(described.last_offset, written.last_offset); + } +} + +/// A create of a closed Stream whose Retention passed fails as expired while its Tombstone is +/// there, before the closed check and before `already_exists`. +async fn a_create_of_an_expired_stream_fails() { + for case in short_cases().await { + case.create().await.unwrap(); + case.close(client()).await.unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + let error = case.create().await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::NotFound, "stream_expired") + ); + } +} + +/// A Producer has a kind, a Workflow appends only through a Local Activity, and a Local Activity's +/// epoch names the Workflow Task that dispatched it. +async fn a_producer_has_a_kind_and_is_not_a_workflow() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let no_kind = Producer { + kind: None, + ..client() + }; + let error = case + .append(no_kind, 1, vec![record("x")]) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::InvalidArgument, "stream_invalid") + ); + let error = case.append(workflow("r1"), 1, vec![]).await.unwrap_err(); + assert_eq!(error.code(), Code::InvalidArgument); + let no_task = Producer { + id: "la:r1:call-1".to_string(), + kind: Some(Kind::LocalActivity(LocalActivityProducer { + run_id: "r1".to_string(), + epoch: Some(LocalActivityEpoch { + task: None, + attempt: 1, + }), + })), + }; + let error = case + .append(no_task, 1, vec![record("x")]) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + } +} + +/// A Producer id is opaque text. The runtime refuses an empty one and one over the limit, and +/// takes any other, also with separators in it. +async fn a_producer_id_is_opaque_but_not_empty_or_too_long() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + for id in [String::new(), "x".repeat(MAX_PRODUCER_ID_BYTES + 1)] { + let wrong = Producer { id, ..client() }; + let error = case.append(wrong, 1, vec![record("x")]).await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::InvalidArgument, "stream_invalid") + ); + } + let odd = Producer { + id: "ingest|eu:1 é".to_string(), + ..client() + }; + case.append(odd.clone(), 1, vec![record("a")]) + .await + .unwrap(); + case.close(odd).await.unwrap(); + } +} + +/// A Producer id may hold any text, such as a NUL, a newline, the `|` and `:` that the Store +/// fields use, and non-ASCII text. An id of the longest length is taken too. A read gives each +/// Record with the id as sent. +async fn a_producer_id_of_any_text_or_the_longest_length_reads_back() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let ids = [ + "nul\0line\nbar|colon:é".to_string(), + "p".repeat(MAX_PRODUCER_ID_BYTES), + ]; + for (sequence, id) in ids.iter().enumerate() { + let producer = Producer { + id: id.clone(), + ..client() + }; + case.append(producer, 1, vec![record(&sequence.to_string())]) + .await + .unwrap(); + } + let read = case.read(ReadRecordsRequest::default()).await.unwrap(); + assert_eq!(texts(&read), ["0", "1"]); + let read_ids: Vec<_> = read + .records + .iter() + .map(|record| record.producer.as_ref().unwrap().id.clone()) + .collect(); + assert_eq!(read_ids, ids); + } +} + +/// A close takes a Producer of any kind. The Workflow Producer sends no epoch, as its close on a +/// Store does. Only an append refuses a Workflow Producer. +async fn a_close_takes_a_producer_of_any_kind() { + let activity = ActivityProducer { + run_id: "r1".to_string(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: Some(prost_types::Timestamp::default()), + attempt: 1, + }), + }; + let kinds = [ + Kind::Workflow(WorkflowProducer { + run_id: "r1".to_string(), + epoch: None, + }), + Kind::LocalActivity(LocalActivityProducer { + run_id: "r1".to_string(), + epoch: Some(LocalActivityEpoch { + task: Some(WorkflowTaskEpoch { + started_event_id: 3, + attempt: 1, + started_time: None, + }), + attempt: 1, + }), + }), + Kind::WorkflowActivity(activity.clone()), + Kind::StandaloneActivity(ActivityProducer { + run_id: String::new(), + ..activity + }), + Kind::Client(ClientProducer {}), + ]; + for case in cases_for(Needs::UNCHECKED).await { + for (at, kind) in kinds.iter().enumerate() { + let case = Case { + ext: case.ext.clone(), + namespace: case.namespace.clone(), + stream: format!("{}-{at}", case.stream), + }; + case.create().await.unwrap(); + let producer = Producer { + id: format!("p:r1:{at}"), + kind: Some(kind.clone()), + }; + case.close(producer.clone()) + .await + .unwrap_or_else(|error| panic!("{producer:?} {error:?}")); + assert_eq!( + case.describe().await.unwrap().status, + StreamStatus::Closed as i32, + "{producer:?}" + ); + } + } +} + +/// An append and a close of a Producer that lacks a required field fail as invalid before the +/// Store, so the Stream gets no Record and stays open. +async fn a_write_with_a_missing_required_field_is_invalid() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let no_epoch = Producer { + id: "la:r1:call-1".to_string(), + kind: Some(Kind::LocalActivity(LocalActivityProducer { + run_id: "r1".to_string(), + epoch: None, + })), + }; + let error = case + .append(no_epoch.clone(), 1, vec![record("x")]) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::InvalidArgument, "stream_invalid") + ); + let error = case.close(no_epoch).await.unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + // A Workflow Producer may close with no epoch, but one that it sends must order. + let mut no_task = workflow("r1"); + let Some(Kind::Workflow(kind)) = no_task.kind.as_mut() else { + unreachable!() + }; + kind.epoch = Some(WorkflowTaskEpoch::default()); + let error = case.close(no_task).await.unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Open as i32); + assert!(describe.last_offset.is_empty()); + } +} + +/// An append with a Record that has no body fails as invalid before the Store, so no Record of the +/// batch is written. +async fn an_append_with_a_record_that_has_no_body_is_invalid() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let records = vec![record("a"), AppendRecord::default()]; + let error = case.append(client(), 1, records).await.unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::InvalidArgument, "stream_invalid") + ); + assert!(case.describe().await.unwrap().last_offset.is_empty()); + } +} + +/// An append with no sequence takes the next one of its Producer's epoch, as a caller that tracks +/// no sequence sends it. Nothing dedupes it, so its retry writes the batch again. A negative +/// sequence fails. +async fn an_append_with_no_sequence_takes_the_next_one_of_its_epoch() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let first = case.append(client(), 0, vec![record("a")]).await.unwrap(); + let retry = case.append(client(), 0, vec![record("a")]).await.unwrap(); + assert!( + retry.first_offset > first.last_offset, + "{first:?} {retry:?}" + ); + let resend = case.append(client(), 2, vec![record("a")]).await.unwrap(); + assert_eq!(resend, retry); + let error = case + .append(client(), -1, vec![record("b")]) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + } +} + +/// A read with no offset starts at the beginning, and a read is exclusive from its offset. A read +/// after the newest offset gives no Record, so a describe and a read give only new Records. An +/// offset past the newest fails with `stream_offset_too_high`, and one that decodes below zero or +/// has more than 8 bytes is invalid. +async fn a_read_is_exclusive_from_its_offset_and_get_offset_reads_only_new_records() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let first = case + .append(client(), 1, vec![record("1"), record("2")]) + .await + .unwrap(); + let all = case.read(ReadRecordsRequest::default()).await.unwrap(); + assert_eq!(texts(&all), ["1", "2"]); + assert_eq!(all.records[0].offset, first.first_offset); + assert_eq!(all.last_offset, first.last_offset); + let from = case + .read(ReadRecordsRequest { + from_offset: first.first_offset.clone(), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(texts(&from), ["2"]); + let newest = case.describe().await.unwrap().last_offset; + let none = case + .read(ReadRecordsRequest { + from_offset: newest.clone(), + ..Default::default() + }) + .await + .unwrap(); + assert!(none.records.is_empty()); + assert_eq!(none.last_offset, newest); + let error = case + .read(ReadRecordsRequest { + from_offset: wire(off(&newest) + 1), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::OutOfRange, "stream_offset_too_high") + ); + let error = case + .read(ReadRecordsRequest { + // An 8-byte value with the top bit set decodes to a negative offset. + from_offset: vec![0x80, 0, 0, 0, 0, 0, 0, 0], + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + let error = case + .read(ReadRecordsRequest { + from_offset: vec![0; 9], + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_invalid"); + // The largest offset has no next one, and it is past the newest Record too. + let error = case + .read(ReadRecordsRequest { + from_offset: wire(i64::MAX), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_offset_too_high"); + } +} + +/// A read of a Stream that nobody created fails at once with `stream_not_found`, also with a long +/// wait. +async fn a_read_of_an_uncreated_stream_fails_at_once() { + for case in cases_for(Needs::CALLS).await { + let started = Instant::now(); + let error = case + .read(ReadRecordsRequest { + wait: millis(5000), + ..Default::default() + }) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::NotFound, "stream_not_found") + ); + assert!(started.elapsed() < Duration::from_secs(2)); + } +} + +/// A quiet read ends empty after its wait, and a long poll wakes on an append. +async fn a_quiet_read_ends_empty_after_its_wait_and_a_long_poll_wakes_on_append() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let first = case.append(client(), 1, vec![record("1")]).await.unwrap(); + let from = first.last_offset; + let started = Instant::now(); + let quiet = case + .read(ReadRecordsRequest { + from_offset: from.clone(), + wait: millis(200), + ..Default::default() + }) + .await + .unwrap(); + assert!(started.elapsed() >= Duration::from_millis(200)); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(quiet.records.is_empty()); + assert_eq!(quiet.last_offset, from); + assert!(!quiet.done); + let ext = case.ext.clone(); + let (namespace, stream) = (case.namespace.clone(), case.stream.clone()); + let appender = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(150)).await; + let request = AppendRecordsRequest { + namespace, + r#ref: Some(StreamRef { + id: stream, + ..Default::default() + }), + producer: Some(client()), + sequence: 2, + records: vec![record("2")], + }; + ext.append_records(request).await + }); + let woke = case + .read(ReadRecordsRequest { + from_offset: from, + wait: millis(5000), + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(texts(&woke), ["2"]); + appender.await.unwrap().unwrap(); + } +} + +/// A read that gives the last Record of a closed Stream is done, and so is every later read, which +/// gives no Record. A close refuses later appends. +async fn a_close_ends_every_read_with_done() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + case.append(client(), 1, vec![record("1")]).await.unwrap(); + case.close(client()).await.unwrap(); + let read = case.read(ReadRecordsRequest::default()).await.unwrap(); + assert_eq!(texts(&read), ["1"]); + assert!(read.done); + let again = case + .read(ReadRecordsRequest { + from_offset: read.last_offset.clone(), + ..Default::default() + }) + .await + .unwrap(); + assert!(again.records.is_empty() && again.done); + assert_eq!(again.last_offset, read.last_offset); + assert_eq!( + case.describe().await.unwrap().status, + StreamStatus::Closed as i32 + ); + let error = case + .append(client(), 2, vec![record("2")]) + .await + .unwrap_err(); + assert_eq!( + (error.code(), reason(&error).as_str()), + (Code::FailedPrecondition, "stream_closed") + ); + } +} + +/// A read of a closed Stream whose Retention passed fails as expired, from any offset, instead +/// of waiting out its wait. +async fn a_read_of_an_expired_closed_stream_fails_instead_of_waiting() { + for case in short_cases().await { + case.create().await.unwrap(); + let first = case.append(client(), 1, vec![record("1")]).await.unwrap(); + case.close(client()).await.unwrap(); + tokio::time::sleep(Duration::from_millis(500)).await; + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Expired as i32); + for request in [ + ReadRecordsRequest::default(), + ReadRecordsRequest { + from_offset: first.first_offset, + ..Default::default() + }, + ReadRecordsRequest { + from_offset: describe.last_offset, + ..Default::default() + }, + ] { + let started = Instant::now(); + let error = case + .read(ReadRecordsRequest { + wait: millis(1500), + ..request + }) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_expired"); + assert!(started.elapsed() < Duration::from_secs(1)); + } + } +} + +/// An open Stream keeps its Records past its Retention, which counts from the close. So a read +/// gets every Record until one Retention after the close, and then the Stream is expired. The +/// Retention is the Store's, since a create takes none. +async fn the_retention_counts_from_the_close() { + for case in short_cases().await { + case.create().await.unwrap(); + let first = case + .append(client(), 1, vec![record("a"), record("b")]) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + let after_first = ReadRecordsRequest { + from_offset: first.first_offset.clone(), + ..Default::default() + }; + let read = case.read(after_first.clone()).await.unwrap(); + assert_eq!(texts(&read), ["b"]); + case.append(client(), 2, vec![record("c")]).await.unwrap(); + case.close(client()).await.unwrap(); + let read = case.read(ReadRecordsRequest::default()).await.unwrap(); + assert_eq!(texts(&read), ["a", "b", "c"]); + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Closed as i32); + tokio::time::sleep(Duration::from_millis(400)).await; + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Expired as i32); + let error = case.read(after_first).await.unwrap_err(); + assert_eq!(reason(&error), "stream_expired"); + } +} + +/// With a last-write TTL in the Store config, an open Stream expires that long after its last +/// write. Each write starts the TTL again. +async fn an_open_stream_expires_its_last_write_ttl_after_its_last_write() { + let config = RuntimeConfig { + last_write_ttl: Some(Duration::from_millis(500)), + ..Default::default() + }; + for case in cases_with(Needs::SHORT_TIMES, &config).await { + case.create().await.unwrap(); + case.append(client(), 1, vec![record("a")]).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + case.append(client(), 2, vec![record("b")]).await.unwrap(); + tokio::time::sleep(Duration::from_millis(300)).await; + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Open as i32); + tokio::time::sleep(Duration::from_millis(500)).await; + let describe = case.describe().await.unwrap(); + assert_eq!(describe.status, StreamStatus::Expired as i32); + let error = case + .append(client(), 3, vec![record("c")]) + .await + .unwrap_err(); + assert_eq!(reason(&error), "stream_expired"); + } +} + +/// A create answer names the configured Store, and the Store never reads the name. +async fn a_create_answer_names_the_store_and_the_store_never_reads_it() { + for ext in super::support::named_runtimes("redis-a") { + let unique = unique(); + let case = Case { + ext, + namespace: format!("ns-{unique}"), + stream: format!("s-{unique}"), + }; + let created = case.create().await.unwrap(); + let stream = created.r#ref.unwrap(); + assert_eq!(stream.id, case.stream); + assert_eq!(stream.store, "redis-a"); + // A ref with any Store name, or none, reaches the same Stream. + let named = |store: &str| { + Some(StreamRef { + id: case.stream.clone(), + store: store.to_string(), + }) + }; + for (sequence, stream) in [(1, named("redis-a")), (2, named("")), (3, named("other"))] { + let request = AppendRecordsRequest { + namespace: case.namespace.clone(), + r#ref: stream, + producer: Some(client()), + sequence, + records: vec![record(&sequence.to_string())], + }; + case.ext.append_records(request).await.unwrap(); + } + let read = ReadRecordsRequest { + namespace: case.namespace.clone(), + r#ref: named("other"), + ..Default::default() + }; + let read = case.ext.read_records(read).await.unwrap(); + assert_eq!(texts(&read), ["1", "2", "3"]); + } + // With no Store name in the config, the ref has none, and lang reads it as the default. + for case in cases_for(Needs::OWN_CONFIG).await { + let created = case.create().await.unwrap(); + assert_eq!(created.r#ref.unwrap().store, ""); + } +} + +/// A read of a closed Stream gives `done` only with its last Record. With no Record left, a read +/// answers at once with `done`, even with a long wait. +async fn a_read_of_a_closed_stream_with_no_record_left_answers_at_once() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let records = vec![record("1"), record("2")]; + let written = case.append(client(), 1, records).await.unwrap(); + case.close(client()).await.unwrap(); + let first = case + .read(ReadRecordsRequest { + max_records: 1, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(texts(&first), ["1"]); + assert!(!first.done); + let rest = case + .read(ReadRecordsRequest { + from_offset: first.last_offset, + ..Default::default() + }) + .await + .unwrap(); + assert_eq!(texts(&rest), ["2"]); + assert!(rest.done); + let started = Instant::now(); + let read = case + .read(ReadRecordsRequest { + from_offset: written.last_offset.clone(), + wait: millis(5000), + ..Default::default() + }) + .await + .unwrap(); + assert!(started.elapsed() < Duration::from_secs(2)); + assert!(read.records.is_empty() && read.done); + assert_eq!(read.last_offset, written.last_offset); + } +} + +/// A long poll at the end of an open Stream wakes on its close and answers with `done`. +async fn a_waiting_read_wakes_with_done_on_a_close() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let written = case.append(client(), 1, vec![record("1")]).await.unwrap(); + let ext = case.ext.clone(); + let close = CloseStreamRequest { + namespace: case.namespace.clone(), + r#ref: case.stream_ref(), + producer: Some(client()), + }; + let closer = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(150)).await; + ext.close_stream(close).await + }); + let started = Instant::now(); + let woke = case + .read(ReadRecordsRequest { + from_offset: written.last_offset, + wait: millis(5000), + ..Default::default() + }) + .await + .unwrap(); + assert!(started.elapsed() < Duration::from_secs(4)); + assert!(woke.records.is_empty() && woke.done); + closer.await.unwrap().unwrap(); + } +} + +/// A page stays under `max_bytes`, and a larger Record is still returned alone. +async fn a_page_keeps_under_max_bytes_and_a_larger_record_comes_alone() { + for case in cases_for(Needs::CALLS).await { + case.create().await.unwrap(); + let big = "x".repeat(1000); + let records = vec![record("a"), record("b"), record(&big), record("c")]; + case.append(client(), 1, records).await.unwrap(); + let capped = |from_offset| ReadRecordsRequest { + from_offset, + max_bytes: 200, + ..Default::default() + }; + let first = case.read(capped(vec![])).await.unwrap(); + assert_eq!(texts(&first), ["a", "b"]); + let second = case.read(capped(first.last_offset)).await.unwrap(); + assert_eq!(texts(&second), [big]); + let third = case.read(capped(second.last_offset)).await.unwrap(); + assert_eq!(texts(&third), ["c"]); + } +} + +/// The cases of the suite, in file order. +pub const CASES: &[super::Case] = &[ + case!(a_create_of_an_open_stream_fails_unless_it_uses_it_or_retries), + case!(a_create_of_a_closed_stream_fails), + case!(a_close_writes_no_record_and_a_second_close_succeeds), + case!(a_create_of_an_expired_stream_fails), + case!(a_producer_has_a_kind_and_is_not_a_workflow), + case!(a_producer_id_is_opaque_but_not_empty_or_too_long), + case!(a_producer_id_of_any_text_or_the_longest_length_reads_back), + case!(a_close_takes_a_producer_of_any_kind), + case!(a_write_with_a_missing_required_field_is_invalid), + case!(an_append_with_a_record_that_has_no_body_is_invalid), + case!(an_append_with_no_sequence_takes_the_next_one_of_its_epoch), + case!(a_read_is_exclusive_from_its_offset_and_get_offset_reads_only_new_records), + case!(a_read_of_an_uncreated_stream_fails_at_once), + case!(a_quiet_read_ends_empty_after_its_wait_and_a_long_poll_wakes_on_append), + case!(a_close_ends_every_read_with_done), + case!(a_read_of_an_expired_closed_stream_fails_instead_of_waiting), + case!(the_retention_counts_from_the_close), + case!(an_open_stream_expires_its_last_write_ttl_after_its_last_write), + case!(a_create_answer_names_the_store_and_the_store_never_reads_it), + case!(a_read_of_a_closed_stream_with_no_record_left_answers_at_once), + case!(a_waiting_read_wakes_with_done_on_a_close), + case!(a_page_keeps_under_max_bytes_and_a_larger_record_comes_alone), +]; diff --git a/crates/streams/src/testing/store.rs b/crates/streams/src/testing/store.rs new file mode 100644 index 000000000..479ab842b --- /dev/null +++ b/crates/streams/src/testing/store.rs @@ -0,0 +1,803 @@ +//! The Store suite. Every Store runs it, so that each one keeps the same rules. +//! +//! A case says its guarantee and what it needs from a Store in its comment. + +use super::{ + case, fields, + support::{Needs, Run, Target, digest, record, records_of, stores_for}, +}; +use crate::{ + Reason, StreamResult, + epoch::{ProducerKind, epoch_key}, + memory::{EntryId, ProducerField}, + offset::{entry_id_of_offset, offset_of_entry_id}, + proto::{ActivityEpoch, ActivityProducer, ClientProducer, Producer, producer::Kind}, + store::{Batch, ConflictPolicy, Create, CreateOutcome, Created}, +}; +use futures_util::future::join_all; +use prost_types::Timestamp; +use std::time::Duration; + +fn activity(run: &str, attempt: i32, scheduled: i64) -> Producer { + Producer { + id: format!("act:{run}:a:s"), + kind: Some(Kind::WorkflowActivity(ActivityProducer { + run_id: run.to_string(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: Some(Timestamp { + seconds: scheduled, + nanos: 0, + }), + attempt, + }), + })), + } +} + +fn client() -> Producer { + Producer { + id: "client:me:p1".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } +} + +/// A second client Producer, which every Store accepts as sent. +fn other_client() -> Producer { + Producer { + id: "client:other:p2".to_string(), + ..client() + } +} + +async fn append( + target: &Target, + producer: &Producer, + sequence: i64, + texts: &[&str], +) -> Result { + let records: Vec<_> = texts.iter().map(|text| record(producer, text)).collect(); + target + .store + .append( + &target.stream_ref(), + &Batch { + producer, + sequence, + records: &records, + digest: digest(texts), + }, + ) + .await +} + +/// The create of a Stream that uses an existing one. +fn use_existing() -> Create<'static> { + Create { + conflict: ConflictPolicy::UseExisting, + ..Default::default() + } +} + +/// A create writes the key once. A create with another Retention finds the Stream and keeps the +/// stored Retention. A create with no request id keeps none. +async fn a_create_writes_the_key_once_and_keeps_its_retention() { + for target in stores_for(Needs::STORE_ANSWERS).await { + assert_eq!( + append(&target, &client(), 1, &["x"]) + .await + .unwrap_err() + .reason, + Reason::NotFound + ); + let day = Create { + retention: Some(Duration::from_secs(86_400)), + ..use_existing() + }; + let created = target.create_with(&day).await.unwrap(); + assert_eq!(created.outcome, CreateOutcome::Created); + let again = target.create_with(&day).await.unwrap(); + assert_eq!(again.outcome, CreateOutcome::Exists); + let other = target.create_with(&use_existing()).await.unwrap(); + assert_eq!(other.outcome, CreateOutcome::Exists); + assert_eq!(other.retention, Duration::from_secs(86_400)); + if target.gives.gives(Needs::RAW_KEYS) { + assert_eq!(target.meta("rid").await, None); + } + } +} + +/// A create of an open Stream fails as `already_exists`, unless it uses the Stream or it is a +/// retry of the create that made the Stream. Such a retry has the same request id, and it succeeds +/// as a create with either policy. The Store keeps that request id as `rid`. +async fn a_create_of_an_open_stream_fails_unless_it_uses_it_or_retries() { + for target in stores_for(Needs::CALLS).await { + let first = Create { + request_id: "create-1", + ..Default::default() + }; + let outcome = |created: StreamResult| created.unwrap().outcome; + let reason = |created: StreamResult| created.unwrap_err().reason; + assert_eq!( + outcome(target.create_with(&first).await), + CreateOutcome::Created + ); + let other = Create { + request_id: "create-2", + ..Default::default() + }; + for create in [&other, &Create::default()] { + assert_eq!( + reason(target.create_with(create).await), + Reason::AlreadyExists + ); + } + let used = Create { + request_id: "create-2", + ..use_existing() + }; + assert_eq!( + outcome(target.create_with(&used).await), + CreateOutcome::Exists + ); + let retry = Create { + request_id: "create-1", + ..use_existing() + }; + for create in [&first, &retry] { + assert_eq!( + outcome(target.create_with(create).await), + CreateOutcome::Created + ); + } + if target.gives.gives(Needs::RAW_KEYS) { + assert_eq!(target.meta("rid").await.as_deref(), Some(&b"create-1"[..])); + } + } +} + +/// A create of an expired or a closed Stream fails as `expired` or `closed`, in this order. These +/// come before the conflict policy and before a retry of the create that made the Stream. +async fn closed_and_expired_come_before_already_exists() { + for target in stores_for(Needs::RAW_KEYS).await { + let first = Create { + request_id: "create-1", + ..Default::default() + }; + target.create_with(&first).await.unwrap(); + let reason = |created: StreamResult| created.unwrap_err().reason; + target.close(&client()).await.unwrap(); + for create in [&first, &Create::default()] { + assert_eq!(reason(target.create_with(create).await), Reason::Closed); + } + target.set_meta("exp", "1").await; + assert_eq!(reason(target.create_with(&first).await), Reason::Expired); + } +} + +/// A resend with the last sequence gets the offsets of its first result, even after another +/// Producer wrote, and a lower sequence is refused. +async fn a_resend_gets_its_first_positions_and_a_lower_sequence_is_refused() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + let first = append(&target, &p, 1, &["p1", "p2"]).await.unwrap(); + // Another Producer writes between the batch and its resend. + append(&target, &other_client(), 1, &["q1"]).await.unwrap(); + let resend = append(&target, &p, 1, &["p1", "p2"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + if target.gives.gives(Needs::STORE_ANSWERS) { + assert!(!first.resend && resend.resend); + } + append(&target, &p, 2, &["p3"]).await.unwrap(); + assert_eq!( + append(&target, &p, 1, &["p1"]).await.unwrap_err().reason, + Reason::SequenceTooLow + ); + let texts = target.texts().await; + assert_eq!(texts, ["p1", "p2", "q1", "p3"]); + } +} + +/// A Producer id is opaque text. An id with the field separator and non-ASCII text keeps its own +/// dedupe state, apart from an id that it starts with. +async fn a_producer_id_is_text_with_its_own_state() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let odd = Producer { + id: "ingest|eu:1 é".to_string(), + ..client() + }; + let plain = Producer { + id: "ingest".to_string(), + ..client() + }; + let first = append(&target, &odd, 1, &["a"]).await.unwrap(); + append(&target, &plain, 1, &["b"]).await.unwrap(); + let resend = append(&target, &odd, 1, &["a"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + assert_eq!(target.texts().await, ["a", "b"]); + } +} + +/// A resend with the last sequence and other content fails as `seq_divergent` and writes nothing. +async fn a_resend_with_other_content_fails_as_divergent() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + append(&target, &p, 1, &["p1"]).await.unwrap(); + let divergent = append(&target, &p, 1, &["other"]).await.unwrap_err(); + assert_eq!(divergent.reason, Reason::SeqDivergent); + assert_eq!(target.texts().await, ["p1"]); + } +} + +/// A lower epoch is fenced for each Producer kind that has an epoch. A client has none, so two +/// writes of one client Producer are never fenced. +async fn a_lower_epoch_is_fenced_for_each_producer_kind() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + let run = Run::new("r1"); + let la = run.local_activity("k", 8, 1, 2); + target.batch(&la, 1, &["la2"]).await.unwrap(); + let stale = target + .batch(&run.local_activity("k", 8, 1, 1), 1, &["la1"]) + .await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + append(&target, &activity("r1", 2, 20), 1, &["a2"]) + .await + .unwrap(); + let stale = append(&target, &activity("r1", 3, 15), 1, &["a3"]).await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + append(&target, &client(), 1, &["c1"]).await.unwrap(); + append(&target, &client(), 2, &["c2"]).await.unwrap(); + assert_eq!(target.texts().await, ["la2", "a2", "c1", "c2"]); + } +} + +/// The fence of a Producer is kept by kind and id, and an epoch has no run. So two runs with their +/// own ids keep their own fences, and a lower epoch of the second run is not fenced. A second run +/// that sends the id of the first shares its fence. +async fn two_runs_with_their_own_ids_keep_their_own_fences() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + let first = Run::new("r1").local_activity("1", 20, 1, 1); + let second = Run::new("r2").local_activity("1", 5, 1, 1); + assert_eq!( + (first.id.as_str(), second.id.as_str()), + ("la:r1:1", "la:r2:1") + ); + target.batch(&first, 1, &["r1"]).await.unwrap(); + target.batch(&second, 1, &["r2"]).await.unwrap(); + let reused = Producer { + id: first.id.clone(), + ..second + }; + let stale = target.batch(&reused, 1, &["reused"]).await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + assert_eq!(target.texts().await, ["r1", "r2"]); + } +} + +/// Two Producers of different kinds with one id keep their own fence state. A high epoch of one +/// fences no lower epoch of another, while a lower epoch of the same kind is still fenced. +async fn producers_of_two_kinds_with_one_id_keep_their_own_fences() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + let id = "shared".to_string(); + let local = Producer { + id: id.clone(), + ..Run::new("r1").local_activity("1", 20, 1, 1) + }; + let high = Producer { + id: id.clone(), + ..activity("r1", 5, 50) + }; + let Some(Kind::WorkflowActivity(epoch)) = activity("", 1, 10).kind else { + unreachable!() + }; + let standalone = Producer { + id: id.clone(), + kind: Some(Kind::StandaloneActivity(epoch)), + }; + target.batch(&local, 1, &["local"]).await.unwrap(); + target.batch(&high, 1, &["activity"]).await.unwrap(); + target.batch(&standalone, 1, &["standalone"]).await.unwrap(); + let lower = Producer { + id, + ..activity("r1", 4, 50) + }; + let stale = target.batch(&lower, 1, &["lower"]).await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + assert_eq!(target.texts().await, ["local", "activity", "standalone"]); + } +} + +/// A close refuses later appends but not a resend of an earlier batch, and a create of the Stream +/// fails as closed. With `STORE_ANSWERS`, a second close returns `already`. +async fn a_close_stops_appends_but_not_a_resend() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + let first = append(&target, &p, 1, &["a"]).await.unwrap(); + let closed = target.close(&p).await.unwrap(); + assert_eq!( + append(&target, &p, 2, &["b"]).await.unwrap_err().reason, + Reason::Closed + ); + assert_eq!( + append(&target, &p, 1, &["a"]).await.unwrap().offsets, + first.offsets + ); + let again = target.close(&p).await.unwrap(); + if target.gives.gives(Needs::STORE_ANSWERS) { + assert!(!closed.already && again.already); + } + let created = target.create_with(&use_existing()).await; + assert_eq!(created.unwrap_err().reason, Reason::Closed); + } +} + +/// A Stream with a newer format version refuses every write. A write checks the version before it +/// reads the Producer's `p|` field, since a newer format can change that field. +async fn a_newer_format_refuses_every_write() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let p = client(); + append(&target, &p, 1, &["a"]).await.unwrap(); + target.set_meta(fields::VERSION, "2").await; + let field = fields::producer(ProducerKind::Client as i32, &p.id); + target.set_meta(&field, "a field of a newer format").await; + let error = append(&target, &p, 2, &["b"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::FormatTooNew); + assert_eq!( + target.close(&p).await.unwrap_err().reason, + Reason::FormatTooNew + ); + let create = target.create_with(&use_existing()).await; + assert_eq!(create.unwrap_err().reason, Reason::FormatTooNew); + } +} + +/// An expired Stream refuses writes and its create. +async fn an_expired_stream_refuses_writes_and_its_create() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + target.set_meta("exp", "1").await; + let error = append(&target, &client(), 1, &["a"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::Expired); + let created = target.create_with(&use_existing()).await; + assert_eq!(created.unwrap_err().reason, Reason::Expired); + } +} + +/// With no last-write TTL, an open Stream's meta and log never expire, and the meta has no `exp`. +/// Its close gives both the Retention, and the meta the Tombstone after it. +async fn an_open_stream_with_no_last_write_ttl_never_expires_until_the_close() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + append(&target, &client(), 1, &["a"]).await.unwrap(); + let week = Duration::from_secs(7 * 86_400).as_millis() as i64; + assert_eq!(target.ttls().await, (-1, -1)); + assert_eq!(target.meta("exp").await, None); + target.close(&client()).await.unwrap(); + let (ttl, meta) = target.ttls().await; + assert!(0 < ttl && ttl <= week, "{ttl}"); + assert!( + meta > Duration::from_secs(36 * 86_400).as_millis() as i64, + "{meta}" + ); + } +} + +/// An open Stream keeps every Record past its Retention, while it is idle and after its next +/// append. No append trims a Record and no read hides one, since the Retention counts from the +/// close. +async fn an_open_stream_keeps_its_records_past_its_retention() { + for target in stores_for(Needs::RAW_KEYS.and(Needs::SHORT_TIMES)).await { + let retention = Duration::from_millis(300); + target + .create_with(&Create { + retention: Some(retention), + ..Default::default() + }) + .await + .unwrap(); + let old = append(&target, &client(), 1, &["old"]).await.unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + assert_eq!(target.texts().await, ["old"]); + append(&target, &client(), 2, &["new"]).await.unwrap(); + let (state, entries) = target + .store + .read(&target.stream_ref(), 0, 10) + .await + .unwrap(); + assert_eq!(state.trimmed, 0); + assert!(state.retained_from() <= old.offsets.unwrap().0); + assert_eq!(records_of(&entries), ["old", "new"]); + // A Store with a last-write TTL gives the log that TTL, and one with none gives no TTL. + let (ttl, _) = target.ttls().await; + assert!(ttl == -1 || ttl > retention.as_millis() as i64, "{ttl}"); + } +} + +/// A write that finds the log gone, as after an eviction, raises the trimmed offset to the meta's +/// newest one. Its offsets go above it, also with the clock behind it. +async fn offsets_go_on_after_the_log_is_gone() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + append(&target, &client(), 1, &["old"]).await.unwrap(); + // A newest id one minute ahead, as after the clock steps back. + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis(); + let ahead = format!("{}-3", now + 60_000); + target.set_meta("newest", &ahead).await; + target.drop_log().await; + assert_eq!(target.ttls().await.0, -2); + let new = append(&target, &client(), 2, &["new"]).await.unwrap(); + let ahead = offset_of_entry_id(&ahead).unwrap(); + assert!(new.offsets.unwrap().0 > ahead); + let (state, _) = target.store.read(&target.stream_ref(), 0, 0).await.unwrap(); + assert_eq!(state.trimmed, ahead); + assert_eq!(target.texts().await, ["new"]); + } +} + +/// A wait for the next entry wakes on an append and ends at its deadline. +async fn a_blocked_read_wakes_on_an_append_and_ends_at_its_deadline() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let first = append(&target, &client(), 1, &["a"]).await.unwrap(); + let newest = first.offsets.unwrap().1; + let deadline = tokio::time::Instant::now() + Duration::from_millis(200); + let woke = target + .store + .wait_after(&target.stream_ref(), newest, deadline) + .await + .unwrap(); + assert!(!woke); + let store = target.store.clone(); + let stream = target.stream_ref(); + let waiter = tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + store.wait_after(&stream, newest, deadline).await + }); + tokio::time::sleep(Duration::from_millis(100)).await; + append(&target, &client(), 2, &["b"]).await.unwrap(); + assert!(waiter.await.unwrap().unwrap()); + } +} + +/// A close writes no log entry. It ends a wait for the next entry, and a wait on a closed Stream +/// ends at once, so a waiting read sees the close. +async fn a_close_writes_no_entry_and_ends_a_wait() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let first = append(&target, &client(), 1, &["a"]).await.unwrap(); + let newest = first.offsets.unwrap().1; + let store = target.store.clone(); + let stream = target.stream_ref(); + let waiter = tokio::spawn(async move { + let deadline = tokio::time::Instant::now() + Duration::from_secs(5); + store.wait_after(&stream, newest, deadline).await + }); + tokio::time::sleep(Duration::from_millis(100)).await; + target.close(&client()).await.unwrap(); + assert!(waiter.await.unwrap().unwrap()); + let started = tokio::time::Instant::now(); + let deadline = started + Duration::from_secs(5); + let woke = target + .store + .wait_after(&target.stream_ref(), newest, deadline) + .await + .unwrap(); + assert!(woke && started.elapsed() < Duration::from_secs(1)); + let (state, entries) = target + .store + .read(&target.stream_ref(), 0, 10) + .await + .unwrap(); + assert!(state.closed); + assert_eq!((state.newest, entries.len()), (newest, 1)); + } +} + +/// The Records of one batch get consecutive offsets that end at the batch's last offset, so a +/// caller knows the offset of each one. +async fn a_batch_gets_consecutive_offsets_that_end_at_its_last_offset() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + append(&target, &client(), 1, &["before"]).await.unwrap(); + // Enough Records that `XADD *` on Redis 7.0 would cross a millisecond. + let texts: Vec = (0..1000).map(|i| i.to_string()).collect(); + let texts: Vec<&str> = texts.iter().map(String::as_str).collect(); + let appended = append(&target, &client(), 2, &texts).await.unwrap(); + let offsets: Vec = target.entries().await[1..] + .iter() + .map(|entry| entry.offset) + .collect(); + assert_eq!(offsets.len(), texts.len()); + assert_eq!( + appended.offsets, + Some((offsets[0], offsets[offsets.len() - 1])) + ); + assert!( + offsets.windows(2).all(|pair| pair[1] == pair[0] + 1), + "{offsets:?}" + ); + } +} + +/// A negative sequence is refused. +async fn a_negative_sequence_is_refused() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let error = append(&target, &client(), -1, &["x"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + assert!(target.texts().await.is_empty()); + } +} + +/// A batch with no sequence takes the next one of its Producer's epoch, and a newer epoch starts +/// at one. Such a batch has no dedupe, so its retry writes it again. A batch that sends the taken +/// number is a resend. +async fn a_batch_with_no_sequence_takes_the_next_one_of_its_epoch() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + let p = activity("r1", 1, 10); + append(&target, &p, 0, &["a"]).await.unwrap(); + let retry = append(&target, &p, 0, &["a"]).await.unwrap(); + let resend = append(&target, &p, 2, &["a"]).await.unwrap(); + assert_eq!(resend.offsets, retry.offsets); + let lower = append(&target, &p, 1, &["a"]).await.unwrap_err(); + assert_eq!(lower.reason, Reason::SequenceTooLow); + append(&target, &p, 7, &["b"]).await.unwrap(); + let next = append(&target, &p, 0, &["c"]).await.unwrap(); + let resend = append(&target, &p, 8, &["c"]).await.unwrap(); + assert_eq!(resend.offsets, next.offsets); + let newer = activity("r1", 2, 10); + let first = append(&target, &newer, 0, &["d"]).await.unwrap(); + let resend = append(&target, &newer, 1, &["d"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + assert_eq!(target.texts().await, ["a", "a", "b", "c", "d"]); + } +} + +/// Batches with no sequence that one Producer epoch sends at once each take their own number, so +/// the Store writes all of them. +async fn batches_with_no_sequence_at_once_each_take_their_own_number() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + let texts = ["t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8"]; + let appends = texts + .iter() + .map(|text| append(&target, &p, 0, std::slice::from_ref(text))); + for appended in join_all(appends).await { + appended.unwrap(); + } + let mut written = target.texts().await; + written.sort(); + assert_eq!(written, texts); + // The batches took 1 to 8, so 8 holds one of them. + let taken = append(&target, &p, 8, &["other"]).await.unwrap_err(); + assert_eq!(taken.reason, Reason::SeqDivergent); + } +} + +/// A large sequence keeps its dedupe entry, so a resend gets its first answer and a lower or +/// divergent batch is refused. A sequence past 2^53 - 1 is refused. +async fn a_large_sequence_dedupes_and_one_past_2_53_is_refused() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let p = client(); + let large = 100_000_000_000_000; + let first = append(&target, &p, large, &["x"]).await.unwrap(); + let resend = append(&target, &p, large, &["x"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + let divergent = append(&target, &p, large, &["other"]).await; + assert_eq!(divergent.unwrap_err().reason, Reason::SeqDivergent); + let lower = append(&target, &p, large - 1, &["lower"]).await; + assert_eq!(lower.unwrap_err().reason, Reason::SequenceTooLow); + assert_eq!(target.texts().await, ["x"]); + let max = (1 << 53) - 1; + append(&target, &p, max, &["max"]).await.unwrap(); + let past = append(&target, &p, max + 1, &["past"]).await; + assert_eq!(past.unwrap_err().reason, Reason::Invalid); + } +} + +/// An open Stream keeps a Producer's dedupe past the Retention. A resend of the last batch of a +/// Producer with no append for longer than the Retention gets the first answer, and the Stream +/// has one copy of the batch. +async fn a_silent_producer_resends_its_last_batch_once_past_the_retention() { + for target in stores_for(Needs::SHORT_TIMES).await { + target + .create_with(&Create { + retention: Some(Duration::from_millis(300)), + ..Default::default() + }) + .await + .unwrap(); + let p = client(); + let first = append(&target, &p, 1, &["old"]).await.unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + // A Store that dropped a silent Producer's state would do it on another Producer's append. + append(&target, &other_client(), 1, &["new"]).await.unwrap(); + let resend = append(&target, &p, 1, &["old"]).await.unwrap(); + assert_eq!(resend.offsets, first.offsets); + if target.gives.gives(Needs::STORE_ANSWERS) { + assert!(resend.resend); + } + assert_eq!(target.texts().await, ["old", "new"]); + } +} + +/// An open Stream keeps a Producer's fence past the Retention, so a lower epoch of a Producer with +/// no append for longer than the Retention is still fenced. +async fn a_silent_producer_stays_fenced_past_the_retention() { + for target in stores_for(Needs::SHORT_TIMES.and(Needs::UNCHECKED)).await { + target + .create_with(&Create { + retention: Some(Duration::from_millis(300)), + ..Default::default() + }) + .await + .unwrap(); + append(&target, &activity("r1", 2, 10), 1, &["new"]) + .await + .unwrap(); + tokio::time::sleep(Duration::from_millis(400)).await; + append(&target, &other_client(), 1, &["other"]) + .await + .unwrap(); + let stale = append(&target, &activity("r1", 1, 10), 1, &["stale"]).await; + assert_eq!(stale.unwrap_err().reason, Reason::Fenced); + assert_eq!(target.texts().await, ["new", "other"]); + } +} + +/// A Producer with a lower epoch can't close the Stream. +async fn a_stale_producer_cant_close_the_stream() { + for target in stores_for(Needs::UNCHECKED).await { + target.create().await; + append(&target, &activity("r1", 2, 10), 1, &["a"]) + .await + .unwrap(); + assert_eq!( + target + .close(&activity("r1", 1, 10)) + .await + .unwrap_err() + .reason, + Reason::Fenced + ); + target.close(&activity("r1", 2, 10)).await.unwrap(); + } +} + +/// A Store refuses an append or a close of a Producer with no epoch, or with an epoch that has no +/// case, since the case gives the kind of the Producer. +async fn a_producer_with_no_epoch_case_is_refused() { + for target in stores_for(Needs::CALLS).await { + target.create().await; + let producer = Producer { + kind: None, + ..client() + }; + let error = append(&target, &producer, 1, &["a"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + let error = target.close(&producer).await.unwrap_err(); + assert_eq!(error.reason, Reason::Invalid); + assert!(target.texts().await.is_empty()); + } +} + +/// An append whose entry ids would not fit an offset is refused before it writes. +async fn an_append_whose_ids_would_not_fit_an_offset_is_refused_before_it_writes() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + // An entry a minute ahead of the clock, with nearly the last 20-bit sequence. + let ahead = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() + + 60_000; + target.add_raw_entry(&format!("{ahead}-1048570")).await; + let error = append(&target, &client(), 1, &["a"; 10]).await.unwrap_err(); + assert_eq!(error.reason, Reason::StorageUnavailable); + assert_eq!(target.log_len().await, 1); + } +} + +/// The entry ids of an append's offsets. +fn entry_ids(offsets: Option<(i64, i64)>) -> Option<(EntryId, EntryId)> { + let id = |offset: i64| { + let text = entry_id_of_offset(offset); + let (ms, seq) = text.split_once('-').unwrap(); + (ms.parse().unwrap(), seq.parse().unwrap()) + }; + offsets.map(|(first, last)| (id(first), id(last))) +} + +/// A Producer's `p|` field has the same 88 bytes on every Store. These are the fence, and the last +/// batch's sequence, entry ids and digest. +async fn a_producer_field_has_the_shared_layout() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let kind = ProducerKind::Activity as i32; + let field = |producer: &Producer, offsets, texts: &[&str]| { + ProducerField { + fence: epoch_key(producer.kind.as_ref()), + sequence: 1, + entries: entry_ids(offsets), + digest: digest(texts), + } + .encode() + }; + let a1 = activity("r1", 1, 10); + let stored = || async { target.meta(fields::producer(kind, &a1.id)).await.unwrap() }; + let written = append(&target, &a1, 1, &["x", "y"]).await.unwrap(); + assert_eq!(stored().await, field(&a1, written.offsets, &["x", "y"])); + // A newer attempt's empty batch sets its fence but writes no entry. + let a2 = activity("r1", 2, 20); + let empty = append(&target, &a2, 1, &[]).await.unwrap(); + assert_eq!(empty.offsets, None); + assert_eq!(stored().await, field(&a2, None, &[])); + } +} + +/// A Store refuses a write of a Producer whose `p|` field has another length, which a later format +/// may write. +async fn a_producer_field_of_another_format_is_refused() { + for target in stores_for(Needs::RAW_KEYS).await { + target.create().await; + let p = client(); + append(&target, &p, 1, &["a"]).await.unwrap(); + let field = fields::producer(ProducerKind::Client as i32, &p.id); + let value = target.meta(&field).await.unwrap(); + let longer = [value.clone(), vec![0]].concat(); + for other in [longer, value[..value.len() - 1].to_vec()] { + target.set_meta(&field, &other).await; + let error = append(&target, &p, 2, &["b"]).await.unwrap_err(); + assert_eq!(error.reason, Reason::StorageCorrupt); + let error = target.close(&p).await.unwrap_err(); + assert_eq!(error.reason, Reason::StorageCorrupt); + } + assert_eq!(target.texts().await, ["a"]); + } +} + +/// The cases of the suite, in file order. +pub const CASES: &[super::Case] = &[ + case!(a_create_writes_the_key_once_and_keeps_its_retention), + case!(a_create_of_an_open_stream_fails_unless_it_uses_it_or_retries), + case!(closed_and_expired_come_before_already_exists), + case!(a_resend_gets_its_first_positions_and_a_lower_sequence_is_refused), + case!(a_resend_with_other_content_fails_as_divergent), + case!(a_producer_id_is_text_with_its_own_state), + case!(a_lower_epoch_is_fenced_for_each_producer_kind), + case!(two_runs_with_their_own_ids_keep_their_own_fences), + case!(producers_of_two_kinds_with_one_id_keep_their_own_fences), + case!(a_close_stops_appends_but_not_a_resend), + case!(a_newer_format_refuses_every_write), + case!(an_expired_stream_refuses_writes_and_its_create), + case!(an_open_stream_with_no_last_write_ttl_never_expires_until_the_close), + case!(an_open_stream_keeps_its_records_past_its_retention), + case!(offsets_go_on_after_the_log_is_gone), + case!(a_blocked_read_wakes_on_an_append_and_ends_at_its_deadline), + case!(a_close_writes_no_entry_and_ends_a_wait), + case!(a_batch_gets_consecutive_offsets_that_end_at_its_last_offset), + case!(a_negative_sequence_is_refused), + case!(a_batch_with_no_sequence_takes_the_next_one_of_its_epoch), + case!(batches_with_no_sequence_at_once_each_take_their_own_number), + case!(a_large_sequence_dedupes_and_one_past_2_53_is_refused), + case!(a_silent_producer_resends_its_last_batch_once_past_the_retention), + case!(a_silent_producer_stays_fenced_past_the_retention), + case!(a_stale_producer_cant_close_the_stream), + case!(a_producer_with_no_epoch_case_is_refused), + case!(an_append_whose_ids_would_not_fit_an_offset_is_refused_before_it_writes), + case!(a_producer_field_has_the_shared_layout), + case!(a_producer_field_of_another_format_is_refused), +]; diff --git a/crates/streams/src/testing/support.rs b/crates/streams/src/testing/support.rs new file mode 100644 index 000000000..3d4d6573a --- /dev/null +++ b/crates/streams/src/testing/support.rs @@ -0,0 +1,301 @@ +//! The helpers that the suites share. They give a new Stream per case on each Store under test, +//! and the Producers of a Workflow run. + +#![allow(dead_code)] + +use super::{RawKeys, RuntimeConfig, test_stores}; +use crate::{ + StreamResult, StreamRuntime, + digest::batch_digest, + proto::{ + AppendRecord, LocalActivityEpoch, LocalActivityProducer, Producer, StreamRecord, StreamRef, + WorkflowTaskEpoch, producer::Kind, + }, + store::{Appended, Batch, Closed, Create, Created, Entry, Store}, +}; +use prost::Message; +use prost_types::Timestamp; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +/// What a case needs beyond the Store's calls on its own Stream. A case runs on every Store that +/// has all that it needs. Each case's comment says its guarantee and its needs. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Needs(u8); + +impl Needs { + /// Only the Store's calls on the Stream and reads of its Records. + pub const CALLS: Self = Self(0); + /// The Store's own results beyond the API. These are a create's stored Retention, `resend`, + /// `flagged`, and a close's offset and `already`. + pub const STORE_ANSWERS: Self = Self(1); + /// The keys under the calls. These are the meta fields, TTLs and raw log entries, and the + /// Store's log calls `read` and `wait_after`. + pub const RAW_KEYS: Self = Self(1 << 1); + /// Producers and runs as sent. The server checks them against its executions, so a + /// case with made-up runs can't run there. + pub const UNCHECKED: Self = Self(1 << 2); + /// A Retention of a few hundred ms. + pub const SHORT_TIMES: Self = Self(1 << 3); + /// The runtime built from the crate's own options, such as a Store name. + pub const OWN_CONFIG: Self = Self(1 << 4); + /// Every need. Redis and memory have all of them. + pub const ALL: Self = Self(u8::MAX); + + /// Both needs. + pub const fn and(self, other: Self) -> Self { + Self(self.0 | other.0) + } + + /// Whether this set gives every need of `needs`. + pub const fn gives(self, needs: Self) -> bool { + self.0 & needs.0 == needs.0 + } +} + +/// One Stream on one server, fresh for each case. +pub(crate) struct Target { + pub(crate) store: Arc, + pub(crate) raw: Arc, + pub(crate) stream: String, + /// What the Store gives. A case checks it before a step that needs more than it asked for. + pub(crate) gives: Needs, +} + +fn unique() -> String { + let n = NEXT.fetch_add(1, Ordering::Relaxed); + format!( + "{}-{n}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + ) +} + +/// A new Stream on each Store under test that gives `needs`. +pub(crate) async fn stores_for(needs: Needs) -> Vec { + let mut targets = vec![]; + for test_store in test_stores() { + if !test_store.gives().gives(needs) { + continue; + } + let (store, raw) = test_store.connect().await; + let unique = unique(); + targets.push(Target { + store, + raw, + stream: format!("wf:{unique}"), + gives: test_store.gives(), + }); + } + targets +} + +/// The Producers of one Workflow run. +pub(crate) struct Run { + pub(crate) id: String, +} + +fn at(seconds: i64) -> Option { + Some(Timestamp { seconds, nanos: 0 }) +} + +impl Run { + pub(crate) fn new(id: &str) -> Self { + Self { id: id.to_string() } + } + + pub(crate) fn local_activity( + &self, + call: &str, + started: i64, + task_attempt: i32, + attempt: i32, + ) -> Producer { + let task = WorkflowTaskEpoch { + started_event_id: started, + attempt: task_attempt, + started_time: at(100), + }; + Producer { + id: format!("la:{}:{call}", self.id), + kind: Some(Kind::LocalActivity(LocalActivityProducer { + run_id: self.id.clone(), + epoch: Some(LocalActivityEpoch { + task: Some(task), + attempt, + }), + })), + } + } +} + +pub(crate) fn record(producer: &Producer, text: &str) -> Vec { + StreamRecord { + producer: Some(producer.clone()), + body: Some( + temporalio_common::protos::temporal::api::common::v1::Payload { + data: text.as_bytes().to_vec(), + ..Default::default() + }, + ), + ..Default::default() + } + .encode_to_vec() +} + +/// The digest of the batch that [record] makes of each text. +pub(crate) fn digest(texts: &[&str]) -> [u8; crate::digest::DIGEST_LEN] { + let records: Vec<_> = texts + .iter() + .map(|text| AppendRecord { + body: Some( + temporalio_common::protos::temporal::api::common::v1::Payload { + data: text.as_bytes().to_vec(), + ..Default::default() + }, + ), + }) + .collect(); + batch_digest(&records) +} + +/// The data texts of the entries, in log order. +pub(crate) fn records_of(entries: &[Entry]) -> Vec { + entries + .iter() + .filter_map(|entry| entry.record.as_ref()) + .map(|bytes| StreamRecord::decode(bytes.as_slice()).unwrap()) + .map(|record| String::from_utf8(record.body.unwrap().data).unwrap()) + .collect() +} + +impl Target { + pub(crate) fn stream_ref(&self) -> StreamRef { + StreamRef { + id: self.stream.clone(), + ..Default::default() + } + } + + pub(crate) async fn create(&self) { + self.create_with(&Create::default()).await.unwrap(); + } + + /// Creates the Stream, and gives the Store's answer. + pub(crate) async fn create_with(&self, create: &Create<'_>) -> StreamResult { + self.store.create(&self.stream_ref(), create).await + } + + /// Sets one meta field under the Store's calls. + pub(crate) async fn set_meta(&self, field: impl AsRef<[u8]>, value: impl AsRef<[u8]>) { + self.raw + .set_meta(&self.stream, field.as_ref(), value.as_ref()) + .await + } + + /// One meta field. + pub(crate) async fn meta(&self, field: impl AsRef<[u8]>) -> Option> { + self.raw.meta(&self.stream, field.as_ref()).await + } + + /// The ms left on the log and on the meta, as `PTTL` gives them. + pub(crate) async fn ttls(&self) -> (i64, i64) { + self.raw.ttls(&self.stream).await + } + + /// Adds a log entry with an explicit id, as `XADD `. + pub(crate) async fn add_raw_entry(&self, id: &str) { + self.raw.add_raw_entry(&self.stream, id).await + } + + /// The number of log entries. + pub(crate) async fn log_len(&self) -> usize { + self.raw.log_len(&self.stream).await + } + + /// Deletes the log, as an eviction would. + pub(crate) async fn drop_log(&self) { + self.raw.drop_log(&self.stream).await + } + + /// Every entry of the log. + pub(crate) async fn entries(&self) -> Vec { + let (_, entries) = self + .store + .read(&self.stream_ref(), 0, 10_000) + .await + .unwrap(); + entries + } + + /// The data texts in the log. + pub(crate) async fn texts(&self) -> Vec { + records_of(&self.entries().await) + } + + /// Appends a batch of `texts` as `producer` with `sequence`. + pub(crate) async fn batch( + &self, + producer: &Producer, + sequence: i64, + texts: &[&str], + ) -> StreamResult { + let records: Vec<_> = texts.iter().map(|text| record(producer, text)).collect(); + self.store + .append( + &self.stream_ref(), + &Batch { + producer, + sequence, + records: &records, + digest: digest(texts), + }, + ) + .await + } + + pub(crate) async fn close(&self, producer: &Producer) -> StreamResult { + self.store.close(&self.stream_ref(), producer).await + } +} + +/// One runtime under test, the namespace its cases use, and what its Store gives. +pub(crate) struct Runtime { + pub(crate) ext: Arc, + /// The namespace of a Store that has fixed ones, such as the server. `None` lets the case + /// make one of its own. + pub(crate) namespace: Option, + pub(crate) gives: Needs, +} + +/// A runtime with `config` on each Store under test that gives `needs`. +pub(crate) async fn runtimes(needs: Needs, config: &RuntimeConfig) -> Vec { + test_stores() + .into_iter() + .filter(|test_store| test_store.gives().gives(needs)) + .map(|test_store| Runtime { + ext: test_store.runtime(config), + namespace: None, + gives: test_store.gives(), + }) + .collect() +} + +/// A runtime on each Store under test, whose refs carry `store_name`. +pub(crate) fn named_runtimes(store_name: &str) -> Vec> { + test_stores() + .into_iter() + .map(|test_store| { + test_store.runtime(&RuntimeConfig { + store_name: store_name.to_string(), + ..Default::default() + }) + }) + .collect() +} diff --git a/crates/streams/testdata/digests.json b/crates/streams/testdata/digests.json new file mode 100644 index 000000000..d870b506a --- /dev/null +++ b/crates/streams/testdata/digests.json @@ -0,0 +1,92 @@ +[ + { + "name": "empty batch", + "records": [], + "digest": "e3b0c44298fc1c149afbf4c8996fb924" + }, + { + "name": "one record, no body", + "records": [ + {} + ], + "digest": "6e340b9cffb37a989ca544e6bb780a2c" + }, + { + "name": "one record, an empty body", + "records": [ + { + "data": "" + } + ], + "digest": "6bb0f5246ac6ee3fa6a6879c512f393f" + }, + { + "name": "one record", + "records": [ + { + "data": "68656c6c6f" + } + ], + "digest": "00cedb65fe85721791cf031124cc4ba2" + }, + { + "name": "metadata in key order", + "records": [ + { + "metadata": { + "encoding": "6a736f6e2f706c61696e", + "a": "00ff" + }, + "data": "7b2278223a317d" + } + ], + "digest": "eea3c073d22bc6bd16ffd5c5a18a2900" + }, + { + "name": "two records", + "records": [ + { + "data": "01" + }, + { + "data": "0203" + } + ], + "digest": "b75e10295b9753dbbc5774e454c9a7fb" + }, + { + "name": "external payloads", + "records": [ + { + "data": "01", + "external_payloads": [ + 10, + 20 + ] + } + ], + "digest": "381530dbc9c2fb0d5ddefbc99e72c6e1" + }, + { + "name": "metadata key and value split one way", + "records": [ + { + "metadata": { + "ab": "63" + } + } + ], + "digest": "be5bd276e8b11daeb753e3d3b6d6f93d" + }, + { + "name": "metadata key and value split another way", + "records": [ + { + "metadata": { + "a": "6263" + } + } + ], + "digest": "ddc7d1a81ecebb12548d3adae06622b8" + } +] diff --git a/crates/streams/testdata/epochs.json b/crates/streams/testdata/epochs.json new file mode 100644 index 000000000..a98d022a4 --- /dev/null +++ b/crates/streams/testdata/epochs.json @@ -0,0 +1,53 @@ +[ + { + "kind": "workflow", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 2, + "task_started_time": [ + 1760000000, + 5 + ] + }, + "key": "0000000000000008000000020000000068e778000000000500000000000000000000000000000000", + "task_key": "0000000000000008000000020000000068e7780000000005" + }, + { + "kind": "local_activity", + "epoch": { + "task_started_event_id": 8, + "task_attempt": 1, + "task_started_time": [ + 1760000000, + 5 + ], + "attempt": 3 + }, + "key": "0000000000000008000000010000000068e778000000000500000000000000000000000000000003", + "task_key": "0000000000000008000000010000000068e7780000000005" + }, + { + "kind": "activity", + "epoch": { + "attempt_scheduled_time": [ + 1760000001, + 999999999 + ], + "attempt": 2 + }, + "key": "0000000000000000000000000000000000000000000000000000000068e778013b9ac9ff00000002", + "task_key": "000000000000000000000000000000000000000000000000" + }, + { + "kind": "client", + "epoch": {}, + "key": "00000000000000000000000000000000000000000000000000000000000000000000000000000000", + "task_key": "000000000000000000000000000000000000000000000000" + }, + { + "kind": "empty", + "epoch": {}, + "key": "00000000000000000000000000000000000000000000000000000000000000000000000000000000", + "task_key": "000000000000000000000000000000000000000000000000" + } +] diff --git a/crates/streams/testdata/fields.json b/crates/streams/testdata/fields.json new file mode 100644 index 000000000..d4427f501 --- /dev/null +++ b/crates/streams/testdata/fields.json @@ -0,0 +1,25 @@ +{ + "kind": 3, + "producer_id": "ingest|eu:1 é", + "producer": "p|3|ingest|eu:1 é", + "producer_values": [ + { + "name": "a batch with Records", + "fence": "0000000000000000000000000000000000000000000000000000000068e778013b9ac9ff00000002", + "sequence": 3, + "first": "1760000000123-4", + "last": "1760000000123-6", + "digest": "aca82bcf079b9b292bba1a09da05f527", + "value": "0000000000000000000000000000000000000000000000000000000068e778013b9ac9ff00000002000000000000000300000199c82cc07b0000000400000199c82cc07b00000006aca82bcf079b9b292bba1a09da05f527" + }, + { + "name": "an empty batch that wrote no entry", + "fence": "0000000000000000000000000000000000000000000000000000000068e778020000000000000003", + "sequence": 1, + "first": "", + "last": "", + "digest": "e3b0c44298fc1c149afbf4c8996fb924", + "value": "0000000000000000000000000000000000000000000000000000000068e7780200000000000000030000000000000001000000000000000000000000000000000000000000000000e3b0c44298fc1c149afbf4c8996fb924" + } + ] +} diff --git a/crates/streams/testdata/ids.json b/crates/streams/testdata/ids.json new file mode 100644 index 000000000..7f85ad77b --- /dev/null +++ b/crates/streams/testdata/ids.json @@ -0,0 +1,25 @@ +{ + "streams": [ + { + "prefix": "temporal-streams", + "stream_id": "agent-1:run-1:turn-3", + "meta": "temporal-streams:{agent-1%3Arun-1%3Aturn-3}:meta", + "log": "temporal-streams:{agent-1%3Arun-1%3Aturn-3}:log", + "closed": "temporal-streams:{agent-1%3Arun-1%3Aturn-3}:closed" + }, + { + "prefix": "p:x", + "stream_id": "a%3Ab%2Fc:r%251:%C3%A9%20name", + "meta": "p%3Ax:{a%253Ab%252Fc%3Ar%25251%3A%25C3%25A9%2520name}:meta", + "log": "p%3Ax:{a%253Ab%252Fc%3Ar%25251%3A%25C3%25A9%2520name}:log", + "closed": "p%3Ax:{a%253Ab%252Fc%3Ar%25251%3A%25C3%25A9%2520name}:closed" + }, + { + "prefix": "temporal-streams", + "stream_id": "{x}:a|b é", + "meta": "temporal-streams:{%7Bx%7D%3Aa%7Cb%20%C3%A9}:meta", + "log": "temporal-streams:{%7Bx%7D%3Aa%7Cb%20%C3%A9}:log", + "closed": "temporal-streams:{%7Bx%7D%3Aa%7Cb%20%C3%A9}:closed" + } + ] +} diff --git a/crates/streams/testdata/offsets.json b/crates/streams/testdata/offsets.json new file mode 100644 index 000000000..c2f529e70 --- /dev/null +++ b/crates/streams/testdata/offsets.json @@ -0,0 +1,22 @@ +[ + { + "entry_id": "0-1", + "offset": 1 + }, + { + "entry_id": "1-0", + "offset": 1048576 + }, + { + "entry_id": "1700000000000-0", + "offset": 1782579200000000000 + }, + { + "entry_id": "1700000000000-7", + "offset": 1782579200000000007 + }, + { + "entry_id": "1760000000123-1048575", + "offset": 1845493760130023423 + } +] diff --git a/crates/streams/tests/capabilities.rs b/crates/streams/tests/capabilities.rs new file mode 100644 index 000000000..addf849ed --- /dev/null +++ b/crates/streams/tests/capabilities.rs @@ -0,0 +1,213 @@ +//! A Store that lacks a capability refuses every call that needs it, by name, and writes nothing. +//! The tests use the memory Store with one capability masked. + +use prost::Message; +use std::{ + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use temporalio_common::protos::temporal::api::common::v1::Payload; +use temporalio_streams::{ + StreamError, StreamResult, StreamRuntime, + memory::MemoryStore, + proto::{ + AppendRecord, AppendRecordsRequest, ClientProducer, CloseStreamRequest, + CreateStreamRequest, DescribeStreamRequest, DescribeStreamResponse, Producer, + StreamErrorDetails, StreamErrorReason, StreamRef, StreamStatus, producer::Kind, + }, + store::{Appended, Batch, Capabilities, Closed, Create, Created, Entry, Store, StreamState}, +}; +use tokio::time::Instant; +use tonic::Code; + +/// A memory Store that lacks the capabilities in `without`, as a Store with fewer would. +struct Masked { + inner: Arc, + without: Capabilities, +} + +#[async_trait::async_trait] +impl Store for Masked { + fn capabilities(&self) -> Capabilities { + self.inner.capabilities().without(self.without) + } + + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult { + self.inner.create(stream_ref, create).await + } + + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult { + self.inner.append(stream_ref, batch).await + } + + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)> { + self.inner.read(stream_ref, from, count).await + } + + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult { + self.inner.wait_after(stream_ref, after, deadline).await + } + + // The masked calls still work underneath, so a call that skips the check would succeed, + // and the tests would see it. + async fn close(&self, stream_ref: &StreamRef, producer: &Producer) -> StreamResult { + self.inner.close(stream_ref, producer).await + } +} + +const NAMESPACE: &str = "ns"; + +struct Case { + ext: Arc, + memory: Arc, + stream: String, +} + +fn case(without: Capabilities) -> Case { + let memory = Arc::new(MemoryStore::new( + Duration::from_secs(3600), + Some(Duration::from_secs(7200)), + )); + let store = Arc::new(Masked { + inner: memory.clone(), + without, + }); + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + Case { + ext: StreamRuntime::new("", store, None), + memory, + stream: format!("wf:run-{unique}"), + } +} + +fn record() -> AppendRecord { + AppendRecord { + body: Some(Payload::default()), + } +} + +fn client() -> Producer { + Producer { + id: "client:me:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } +} + +/// The capability an error names, after it checked the code and the reason. +fn refused_capability(error: &StreamError) -> String { + assert_eq!(error.code(), Code::Unimplemented, "{error:?}"); + let details = StreamErrorDetails::decode(error.details().as_slice()).unwrap(); + assert_eq!(details.reason(), StreamErrorReason::Unsupported); + error + .message + .rsplit(' ') + .next() + .unwrap_or_default() + .to_string() +} + +impl Case { + fn stream_ref(&self) -> Option { + Some(StreamRef { + id: self.stream.clone(), + ..Default::default() + }) + } + + async fn call( + &self, + operation: &str, + request: Req, + ) -> Result, StreamError> { + self.ext.call(operation, &request.encode_to_vec()).await + } + + async fn create(&self) { + let request = CreateStreamRequest { + namespace: NAMESPACE.to_string(), + stream_ref: self.stream_ref(), + ..Default::default() + }; + self.call("CreateStream", request).await.unwrap(); + } + + fn log_len(&self) -> usize { + self.memory.test_log_len(&self.stream) + } +} + +#[tokio::test] +async fn an_append_without_the_fence_is_refused_with_nothing_written() { + let case = case(Capabilities::PRODUCER_FENCE); + case.create().await; + let request = AppendRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: case.stream_ref(), + producer: Some(client()), + sequence: 1, + records: vec![record()], + }; + let error = case.call("AppendRecords", request).await.unwrap_err(); + assert_eq!(refused_capability(&error), "producer_fence"); + assert_eq!(case.log_len(), 0); +} + +#[tokio::test] +async fn an_append_with_no_sequence_without_the_capability_is_refused_with_nothing_written() { + let case = case(Capabilities::NEXT_SEQUENCE); + case.create().await; + let request = AppendRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: case.stream_ref(), + producer: Some(client()), + records: vec![record()], + ..Default::default() + }; + let error = case + .call("AppendRecords", request.clone()) + .await + .unwrap_err(); + assert_eq!(refused_capability(&error), "next_sequence"); + assert_eq!(case.log_len(), 0); + // A batch with its own sequence needs no number from the Store. + let numbered = AppendRecordsRequest { + sequence: 1, + ..request + }; + case.call("AppendRecords", numbered).await.unwrap(); + assert_eq!(case.log_len(), 1); +} + +#[tokio::test] +async fn a_close_without_the_capability_is_refused_and_the_stream_stays_open() { + let case = case(Capabilities::CLOSE); + case.create().await; + let request = CloseStreamRequest { + namespace: NAMESPACE.to_string(), + r#ref: case.stream_ref(), + producer: Some(client()), + }; + let error = case.call("CloseStream", request).await.unwrap_err(); + assert_eq!(refused_capability(&error), "close"); + assert_eq!(case.log_len(), 0); + let describe = DescribeStreamRequest { + namespace: NAMESPACE.to_string(), + r#ref: case.stream_ref(), + }; + let bytes = case.call("DescribeStream", describe).await.unwrap(); + let described = DescribeStreamResponse::decode(bytes.as_slice()).unwrap(); + assert_eq!(described.status, StreamStatus::Open as i32); +} diff --git a/crates/streams/tests/scans.rs b/crates/streams/tests/scans.rs new file mode 100644 index 000000000..f1a52610b --- /dev/null +++ b/crates/streams/tests/scans.rs @@ -0,0 +1,206 @@ +//! How a read scans the log, seen through a memory Store that can trim under a read. + +use prost::Message; +use std::{ + sync::{ + Arc, + atomic::{AtomicBool, AtomicUsize, Ordering}, + }, + time::Duration, +}; +use temporalio_common::protos::temporal::api::common::v1::Payload; +use temporalio_streams::{ + StreamError, StreamResult, StreamRuntime, + memory::MemoryStore, + proto::{ + AppendRecord, AppendRecordsRequest, AppendRecordsResponse, ClientProducer, + CreateStreamRequest, CreateStreamResponse, Producer, ReadRecordsRequest, + ReadRecordsResponse, StreamErrorDetails, StreamErrorReason, StreamRef, producer::Kind, + }, + store::{Appended, Batch, Capabilities, Create, Created, Entry, Store, StreamState}, +}; +use tokio::time::Instant; + +const NAMESPACE: &str = "ns"; +const STREAM: &str = "s"; + +/// The memory Store. Once armed, its next read that asks for log entries finds its first entry +/// trimmed, as a trim just before that read leaves it. It counts the entries that reads return. +struct Trimming { + inner: MemoryStore, + trim_next: AtomicBool, + pulled: AtomicUsize, +} + +#[async_trait::async_trait] +impl Store for Trimming { + fn capabilities(&self) -> Capabilities { + self.inner.capabilities() + } + + async fn create(&self, stream_ref: &StreamRef, create: &Create<'_>) -> StreamResult { + self.inner.create(stream_ref, create).await + } + + async fn append(&self, stream_ref: &StreamRef, batch: &Batch<'_>) -> StreamResult { + self.inner.append(stream_ref, batch).await + } + + async fn read( + &self, + stream_ref: &StreamRef, + from: i64, + count: usize, + ) -> StreamResult<(StreamState, Vec)> { + let (mut state, mut entries) = self.inner.read(stream_ref, from, count).await?; + if count > 0 && !entries.is_empty() && self.trim_next.swap(false, Ordering::Relaxed) { + state.trimmed = entries.remove(0).offset; + } + self.pulled.fetch_add(entries.len(), Ordering::Relaxed); + Ok((state, entries)) + } + + async fn wait_after( + &self, + stream_ref: &StreamRef, + after: i64, + deadline: Instant, + ) -> StreamResult { + self.inner.wait_after(stream_ref, after, deadline).await + } +} + +fn trimming() -> (Arc, Arc) { + let store = Arc::new(Trimming { + inner: MemoryStore::new(Duration::from_secs(3600), Some(Duration::from_secs(7200))), + trim_next: AtomicBool::new(false), + pulled: AtomicUsize::new(0), + }); + let ext = StreamRuntime::new("", store.clone(), None); + (store, ext) +} + +async fn call( + ext: &StreamRuntime, + operation: &str, + request: Req, +) -> Resp { + try_call(ext, operation, request).await.unwrap() +} + +async fn try_call( + ext: &StreamRuntime, + operation: &str, + request: Req, +) -> Result { + let bytes = ext.call(operation, &request.encode_to_vec()).await?; + Ok(Resp::decode(bytes.as_slice()).unwrap()) +} + +async fn create(ext: &StreamRuntime) { + let _: CreateStreamResponse = call( + ext, + "CreateStream", + CreateStreamRequest { + namespace: NAMESPACE.to_string(), + stream_ref: stream_ref(), + ..Default::default() + }, + ) + .await; +} + +fn client() -> Producer { + Producer { + id: "client:me:p".to_string(), + kind: Some(Kind::Client(ClientProducer {})), + } +} + +fn stream_ref() -> Option { + Some(StreamRef { + id: STREAM.to_string(), + ..Default::default() + }) +} + +async fn append(ext: &StreamRuntime, sequence: i64, count: usize) -> AppendRecordsResponse { + append_sized(ext, sequence, count, 0).await +} + +/// Appends `count` Records, each with `size` bytes after its number. +async fn append_sized( + ext: &StreamRuntime, + sequence: i64, + count: usize, + size: usize, +) -> AppendRecordsResponse { + let records = (0..count) + .map(|i| AppendRecord { + body: Some(Payload { + data: format!("{i}{}", "x".repeat(size)).into_bytes(), + ..Default::default() + }), + }) + .collect(); + call( + ext, + "AppendRecords", + AppendRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: stream_ref(), + producer: Some(client()), + sequence, + records, + }, + ) + .await +} + +/// A trim between a read's state and its log read fails a read from an offset as expired, so that +/// the Reader always sees the gap. A read from the start skips the trimmed Records. +#[tokio::test] +async fn a_trim_under_a_read_from_an_offset_fails_it_as_expired() { + let (store, ext) = trimming(); + create(&ext).await; + let first = append(&ext, 1, 3).await; + let read = |from_offset| ReadRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: stream_ref(), + from_offset, + ..Default::default() + }; + store.trim_next.store(true, Ordering::Relaxed); + let error = try_call::<_, ReadRecordsResponse>(&ext, "ReadRecords", read(first.first_offset)) + .await + .unwrap_err(); + let details = StreamErrorDetails::decode(error.details().as_slice()).unwrap(); + assert_eq!(details.reason(), StreamErrorReason::Expired); + store.trim_next.store(true, Ordering::Relaxed); + let from_start: ReadRecordsResponse = call(&ext, "ReadRecords", read(vec![])).await; + assert_eq!(from_start.records.len(), 2); +} + +/// A read with a byte cap reads about the entries that its page holds from the Store, so a small +/// cap on large Records doesn't read a whole page of them. +#[tokio::test] +async fn a_capped_read_pulls_about_its_page_from_the_store() { + let (store, ext) = trimming(); + create(&ext).await; + append_sized(&ext, 1, 50, 1000).await; + store.pulled.store(0, Ordering::Relaxed); + let page: ReadRecordsResponse = call( + &ext, + "ReadRecords", + ReadRecordsRequest { + namespace: NAMESPACE.to_string(), + r#ref: stream_ref(), + max_bytes: 2500, + ..Default::default() + }, + ) + .await; + assert_eq!(page.records.len(), 2); + let pulled = store.pulled.load(Ordering::Relaxed); + assert!(pulled <= 4, "{pulled}"); +} diff --git a/crates/streams/tests/vectors.rs b/crates/streams/tests/vectors.rs new file mode 100644 index 000000000..f1a42ae26 --- /dev/null +++ b/crates/streams/tests/vectors.rs @@ -0,0 +1,155 @@ +//! The shared test vectors, which `examples/gen_vectors.rs` writes from a second implementation. +//! The crate's README says which other side reads each file. + +use prost_types::Timestamp; +use serde_json::Value; +use temporalio_common::protos::temporal::api::common::v1::{ + Payload, payload::ExternalPayloadDetails, +}; +use temporalio_streams::{ + digest::batch_digest, + epoch::{epoch_key, task_key, task_of}, + ids::StreamKeys, + offset::{entry_id_of_offset, offset_of_entry_id}, + proto::{ + ActivityEpoch, ActivityProducer, AppendRecord, ClientProducer, LocalActivityEpoch, + LocalActivityProducer, WorkflowProducer, WorkflowTaskEpoch, producer::Kind, + }, +}; + +fn load(name: &str) -> Value { + let path = format!("{}/testdata/{name}", env!("CARGO_MANIFEST_DIR")); + serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap() +} + +fn text<'a>(value: &'a Value, field: &str) -> &'a str { + value[field].as_str().unwrap() +} + +fn time(value: &Value) -> Option { + let pair = value.as_array()?; + Some(Timestamp { + seconds: pair[0].as_i64().unwrap(), + nanos: pair[1].as_i64().unwrap() as i32, + }) +} + +fn number(value: &Value, field: &str) -> i64 { + value[field].as_i64().unwrap_or(0) +} + +/// The Producer kind of a vector, from its flat fields and its run. The case takes the fields of +/// its kind, and a run-based kind carries the run. `empty` has no case. +fn kind_of(kind: &str, value: &Value, run_id: &str) -> Option { + let task = WorkflowTaskEpoch { + started_event_id: number(value, "task_started_event_id"), + attempt: number(value, "task_attempt") as i32, + started_time: time(&value["task_started_time"]), + }; + let attempt = number(value, "attempt") as i32; + let run_id = run_id.to_string(); + let activity = ActivityProducer { + run_id: run_id.clone(), + epoch: Some(ActivityEpoch { + attempt_scheduled_time: time(&value["attempt_scheduled_time"]), + attempt, + }), + }; + match kind { + "workflow" => Some(Kind::Workflow(WorkflowProducer { + run_id, + epoch: Some(task), + })), + "local_activity" => Some(Kind::LocalActivity(LocalActivityProducer { + run_id, + epoch: Some(LocalActivityEpoch { + task: Some(task), + attempt, + }), + })), + "activity" => Some(Kind::WorkflowActivity(activity)), + "standalone_activity" => Some(Kind::StandaloneActivity(activity)), + "client" => Some(Kind::Client(ClientProducer {})), + "empty" => None, + other => panic!("unknown kind {other}"), + } +} + +#[test] +fn stream_ids_and_keys_match() { + let vectors = load("ids.json"); + for case in vectors["streams"].as_array().unwrap() { + let keys = StreamKeys::new(text(case, "prefix"), text(case, "stream_id")); + assert_eq!(keys.meta(), text(case, "meta")); + assert_eq!(keys.log(), text(case, "log")); + assert_eq!(keys.closed(), text(case, "closed")); + } +} + +#[test] +fn offsets_match() { + for case in load("offsets.json").as_array().unwrap() { + let offset = number(case, "offset"); + assert_eq!(offset_of_entry_id(text(case, "entry_id")).unwrap(), offset); + assert_eq!(entry_id_of_offset(offset), text(case, "entry_id")); + } +} + +#[test] +fn epoch_keys_match() { + for case in load("epochs.json").as_array().unwrap() { + let kind = kind_of(text(case, "kind"), &case["epoch"], ""); + assert_eq!(epoch_key(kind.as_ref()).to_vec(), hex(text(case, "key"))); + assert_eq!( + task_key(&task_of(kind.as_ref())).to_vec(), + hex(text(case, "task_key")) + ); + } +} + +fn hex(text: &str) -> Vec { + (0..text.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&text[i..i + 2], 16).unwrap()) + .collect() +} + +#[test] +fn batch_digests_match() { + for case in load("digests.json").as_array().unwrap() { + let records: Vec<_> = case["records"] + .as_array() + .unwrap() + .iter() + .map(|r| AppendRecord { + // A record with none of these keys has no body. + body: ["metadata", "data", "external_payloads"] + .iter() + .any(|key| r.get(key).is_some()) + .then(|| Payload { + metadata: r["metadata"] + .as_object() + .into_iter() + .flatten() + .map(|(key, value)| (key.clone(), hex(value.as_str().unwrap()))) + .collect(), + data: hex(r["data"].as_str().unwrap_or_default()), + external_payloads: r["external_payloads"] + .as_array() + .into_iter() + .flatten() + .map(|size| ExternalPayloadDetails { + size_bytes: size.as_i64().unwrap(), + }) + .collect(), + }), + }) + .collect(); + assert_eq!( + batch_digest(&records).to_vec(), + hex(text(case, "digest")), + "{}", + text(case, "name") + ); + } +}