From 3332d2d93e53fd0866d30d39a414f78288ddd594 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Fri, 6 Nov 2026 02:43:44 -0700 Subject: [PATCH 1/7] Added a chat app that shows most parts of the Streams API. --- tests/contrib/streams/examples/README.md | 3 + .../streams/examples/chat_app/.gitignore | 1 + .../streams/examples/chat_app/README.md | 204 +++ .../streams/examples/chat_app/__init__.py | 0 .../streams/examples/chat_app/activities.py | 65 + .../contrib/streams/examples/chat_app/docs.py | 67 + .../examples/chat_app/models/__init__.py | 69 + .../examples/chat_app/models/claude.py | 153 ++ .../streams/examples/chat_app/models/fake.py | 131 ++ .../examples/chat_app/models/openai_chat.py | 106 ++ .../streams/examples/chat_app/pyproject.toml | 19 + .../streams/examples/chat_app/server.py | 493 +++++++ .../streams/examples/chat_app/shared.py | 191 +++ .../examples/chat_app/static/index.html | 1262 +++++++++++++++++ .../examples/chat_app/test_chat_app.py | 339 +++++ .../streams/examples/chat_app/tools.py | 170 +++ .../streams/examples/chat_app/workflow.py | 333 +++++ 17 files changed, 3606 insertions(+) create mode 100644 tests/contrib/streams/examples/chat_app/.gitignore create mode 100644 tests/contrib/streams/examples/chat_app/README.md create mode 100644 tests/contrib/streams/examples/chat_app/__init__.py create mode 100644 tests/contrib/streams/examples/chat_app/activities.py create mode 100644 tests/contrib/streams/examples/chat_app/docs.py create mode 100644 tests/contrib/streams/examples/chat_app/models/__init__.py create mode 100644 tests/contrib/streams/examples/chat_app/models/claude.py create mode 100644 tests/contrib/streams/examples/chat_app/models/fake.py create mode 100644 tests/contrib/streams/examples/chat_app/models/openai_chat.py create mode 100644 tests/contrib/streams/examples/chat_app/pyproject.toml create mode 100644 tests/contrib/streams/examples/chat_app/server.py create mode 100644 tests/contrib/streams/examples/chat_app/shared.py create mode 100644 tests/contrib/streams/examples/chat_app/static/index.html create mode 100644 tests/contrib/streams/examples/chat_app/test_chat_app.py create mode 100644 tests/contrib/streams/examples/chat_app/tools.py create mode 100644 tests/contrib/streams/examples/chat_app/workflow.py diff --git a/tests/contrib/streams/examples/README.md b/tests/contrib/streams/examples/README.md index bc2dea4f46..1109fe2d0e 100644 --- a/tests/contrib/streams/examples/README.md +++ b/tests/contrib/streams/examples/README.md @@ -13,3 +13,6 @@ end to end on the memory Store and on Redis. | Worker setup | [`worker_setup.py`](worker_setup.py) | The proposal's Nexus examples aren't in this directory. + +[`chat_app`](chat_app/README.md) is a small chat app that uses most parts of the API, with a page +that shows each Stream item as it arrives. diff --git a/tests/contrib/streams/examples/chat_app/.gitignore b/tests/contrib/streams/examples/chat_app/.gitignore new file mode 100644 index 0000000000..07df930ad7 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/.gitignore @@ -0,0 +1 @@ +uv.lock diff --git a/tests/contrib/streams/examples/chat_app/README.md b/tests/contrib/streams/examples/chat_app/README.md new file mode 100644 index 0000000000..3fc9615bcc --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/README.md @@ -0,0 +1,204 @@ +# Streams chat app + +A small ChatGPT-like app that uses most parts of the Streams API. Each chat is a Workflow with an +agent that answers in rounds. A round is a model call, which can call tools, and the tools' results +go to the next round. Each round streams its token deltas to a tokens Stream, and the turns Stream +gets a Turn for each round, each tool call and each tool result. A FastAPI server reads the Streams +and sends them to the browser as Server-Sent Events. The page shows the chat, and a log of each +Stream item with its offset, its Producer and its kind. + +## Run it + +From the repo root, with the SDK's dev environment. + +```sh +uv sync --all-extras +poe build-develop + +# In two other shells. Redis is optional, see below. +temporal server start-dev --port 7245 --ui-port 8245 +redis-server --port 6681 --save "" --appendonly no + +export TEMPORAL_ADDRESS=localhost:7245 +export CHAT_APP_REDIS_URL=redis://127.0.0.1:6681 +uv run uvicorn tests.contrib.streams.examples.chat_app.server:app --port 8010 \ + --timeout-graceful-shutdown 1 +``` + +Then open . Each open tab holds a connection, so without +`--timeout-graceful-shutdown` a stop of the server waits for the tabs to close. + +The server runs the Worker in its own process. Without `CHAT_APP_REDIS_URL`, it uses the +in-memory backend, and the Streams go away when the server stops. + +### Models + +The page header shows the provider and the model. `CHAT_MODEL_PROVIDER` picks the provider. It +takes `claude`, `openai` or `fake`. Without it, the server takes Claude if `ANTHROPIC_API_KEY` is +set, else OpenAI if `OPENAI_API_KEY` is set, else the scripted model. + +- **Claude.** Export `ANTHROPIC_API_KEY`. The model is `claude-opus-5-5`, and `ANTHROPIC_MODEL` + picks another. Each round is one streamed request with tool use and low effort. The next + round sends back the round's blocks unchanged, so Claude keeps its thinking. A round that is + refused, or cut off in a tool call, runs no tool. The call opts in to the server-side refusal + fallback (`fallbacks="default"`), so a declined request runs again on the model that Anthropic + picks for it. If the whole chain declines, the page shows a short notice instead of the reply. + With `CHAT_MODEL_PROVIDER=claude`, the `anthropic` SDK can also use other credentials, such as + a profile from `ant auth login`. +- **OpenAI.** Export `OPENAI_API_KEY`. The model is `gpt-4o-mini`, and `OPENAI_MODEL` picks + another. Each round is one streamed request with tool calls. +- **Scripted.** It plans each round from the prompt and the tool results so far, so the + scenarios take the same rounds with no key. The tests use it. + +The tools are local and make no network call. They are `calculator`, `clock`, `roll_dice`, +`search_docs`, which searches a short text about Streams in [`docs.py`](docs.py), and `research`, +which starts a sub-agent. Each tool works for a moment, so tools that run at the same time +interleave their Turns. + +Each provider gets only the system prompt and the chat's messages. The app reads no key itself, +and a failed call keeps only the error's class, since an API error's text can quote the request. + +To run the app in its own environment instead, build it from [`pyproject.toml`](pyproject.toml). +It takes the SDK from this repo, so the build needs the Stream runtime. + +```sh +MATURIN_PEP517_ARGS="--features streams" uv sync --project tests/contrib/streams/examples/chat_app +uv run --project tests/contrib/streams/examples/chat_app \ + uvicorn tests.contrib.streams.examples.chat_app.server:app --port 8010 \ + --timeout-graceful-shutdown 1 +``` + +## Scenarios + +The bar above the composer has a preset prompt for each case. "Run all" plays them one after +another in one chat, so a new viewer sees each case. + +- **Tool rounds.** Three model rounds, with `roll_dice` and then `calculator` between them. Each + round streams its own tokens, and each tool gets a card with its Turns. +- **Parallel tools.** One round calls `clock` and `calculator`, and the two tool Activities write + to the turns Stream at the same time. The log shows two Activity Producers. Each Producer's + Turns keep their order, and the two interleave in the order that the Store got them, which + every reader sees. +- **Failed attempt.** The first attempt of the round fails after a few tokens. One second later, + the retry's `Superseded` item crosses them out, and the round streams again. +- **Message during a run.** While `search_docs` runs, the page sends a second message. The + `add_message` Signal publishes its Turn at once with `publish_nowait`, and the message joins + the next round of the same turn. While a turn runs, the composer sends this way too. +- **Sub-agent.** `research` starts a child Workflow. It creates its own Streams, under ids with + its run id that the parent can't know, and writes a Turn with their refs to the parent's turns + Stream. The page opens a panel that reads the child's Streams by those ids. +- **Continue-as-new.** Six rolls in six rounds, then the answer. After six rounds, the Workflow + continues as new between turns, and the log shows the next run on the same Streams. +- **Share.** A snapshot of the chat so far. + +## What to try + +- Send a message, and watch the rounds stream token by token. Each token is a row in the log. +- Turn on "Fail the first attempt of the next reply", and send a message. The model Activity fails + after a few tokens. One second later, the retry's `Superseded` item crosses out the failed + attempt's tokens, and the round streams again. +- Stop the server in the middle of a reply, and start it again with Redis. The attempt misses its + heartbeats, and its retry withdraws its tokens. The tab reconnects, and the server checks the + tokens that the tab kept with `withdraws()`, so the page crosses them out. +- Reload the page, also in the middle of a reply. The tab keeps its place in each Stream and + resumes from it. +- Click "Open a second reader". The new tab reads the turns from the start and the tokens from + the end. +- Click "Share a snapshot". The server copies the chat into a new Stream and closes it. The link + reads that Stream to its end. The snapshot's offsets and Producer are its own, so each Record + is a `SharedTurn`, which keeps the Turn's offset and Producer in the chat. The log shows both. + A link from an older version of the app, whose id has `-share-` and not `-snapshot-`, asks you + to share the chat again. +- Click "End chat". The Workflow closes its Streams, and each read ends. + +## Where each feature is + +- **`create_or_get_stream` in a Workflow.** [`ChatSession.__init__`](workflow.py) gets the turns + and the tokens Stream with `workflow.create_or_get_stream_nowait[...]`. A chat id can come back, + such as in a restarted session, and the run then gets the chat's Streams. +- **`create_or_get_stream` in a client.** The server's [`lifespan`](server.py) gets the lobby + Stream, which lists the chats. Each server start runs it. +- **Strict create.** [`share`](server.py) creates each snapshot with + `client.create_stream[SharedTurn]`, since a share id names one snapshot. A second share of the + same point raises `StreamClosedError`, since that snapshot is closed. A share that runs at the + same time raises `StreamAlreadyExistsError`. Both give the same link. +- **Standalone Streams that a Workflow creates.** `-turns` and `-tokens`, with ids + that the Workflow picks. +- **Refs as data.** The `streams` query gives `SessionStreams`. `ModelInput.tokens` takes a ref to + the Activity, and `SessionInput.streams` takes both refs through continue-as-new. +- **`to_stream`.** `workflow.to_stream` in the next run, `activity.client().to_stream` in the + Activity, and `client.to_stream(ref, payload_type=Turn)` in the server. +- **`client.get_stream[T]` by id.** The share links, which name a snapshot's Stream id. +- **Awaited `publish`.** The `send` Update returns the user Turn's offset, which the page shows + under the composer. The server's lobby publish is awaited too. +- **`publish_nowait`.** `@workflow.init` publishes "Run N of the session started", and the + `add_message` Signal publishes the user's Turn, since neither can await. +- **An agent loop.** [`Agent.answer`](workflow.py) runs the rounds of an answer, for a chat and + for a sub-agent. Each round is the model Activity, and the round's tools run at the same time. +- **Activities that write Turns.** [`run_tool`](tools.py) writes its progress and its result to + the turns Stream, from a ref in its input. Each attempt of each tool is its own Producer. +- **A child Workflow's own Streams.** [`Researcher`](workflow.py) creates its Streams at run time, + writes to its parent's turns Stream through `workflow.to_stream`, and puts its own refs in a + Turn. A ref is data, also inside a Record. +- **`BufferedPublisher` and `flush`.** [`call_model`](activities.py) batches the deltas, and + flushes before the toggle fails the attempt. +- **`subscribe(offset=...)`.** [`session_events`](server.py) resumes each read at the offset that + the tab saved. +- **`streams.END`.** A tab that joins reads only new tokens, since the turns Stream has the + earlier replies. +- **`subscribe(last=...)`.** The lobby sidebar reads the newest 20 chats, then each new one. +- **`create_wait`.** A new chat's tab can start to read before the init's creates. +- **Resume from `next_offset`.** Each item that the server sends carries its read's + `next_offset`. The tab saves it, and a reload or a reconnect sends it back. +- **`Superseded` withdrawal.** The page uses `withdrawn_offsets` for tokens that the read gave, + and the server uses `withdraws()` for tokens that the tab kept from an earlier read. +- **Close with `async with`.** The share snapshot's block in the server, and the sub-agent's + Streams in `Researcher`. +- **`close()`.** `ChatSession.run` closes both Streams when the chat ends, since a run's end + doesn't close a Stream. +- **`get_offset`.** The snapshot id names the turns Stream's offset. The share page gets the + snapshot's offset, and an unknown snapshot gives `StreamNotFoundError`. +- **Error classes.** `StreamClosedError` and `StreamAlreadyExistsError` in `share`, and + `StreamNotFoundError` in `share_info`. +- **Payload type.** `[Turn]`, `[Token]`, `[SharedTurn]` and `[SessionInfo]` on the creates and + gets, and `payload_type=` on `client.to_stream`. +- **`client.get_stream[T]` by id for a sub-agent.** The server reads a sub-agent's Streams by the + ids in its Turn, for the page's panel. +- **In-memory backend.** The tests run on it and on Redis. The server uses it when + `CHAT_APP_REDIS_URL` isn't set. +- **Producers.** The log shows each item's Producer. The Workflow writes through Local + Activities, each model or tool attempt is an Activity Producer, a sub-agent writes through its + own Local Activities, and the server writes as a client. + +The app doesn't use `Stream.producer()`, `SyncStream`, `close_nowait` or `StreamFencedError`. + +## Coming + +These are approved but not built yet, so the app doesn't use them. + +- A `closed` item with a close status. The log would show why a Stream closed. +- `max_record_age` on a read. +- Opaque offsets. The app already passes offsets to the browser as strings. + +The Python facade has no public `describe` yet, so the app uses only `get_offset`. + +## Files + +- [`shared.py`](shared.py) has the Records and the inputs that the Workflow, the Activity and the + server share. +- [`workflow.py`](workflow.py) has the agent loop, the `ChatSession` Workflow and the + `Researcher` sub-agent. +- [`activities.py`](activities.py) has the model Activity. +- [`tools.py`](tools.py) has the tools and their Activity, and [`docs.py`](docs.py) the text + that `search_docs` searches. +- [`models/`](models/__init__.py) has one module for each provider, behind one interface. +- [`server.py`](server.py) has the FastAPI server and its Server-Sent Events. +- [`static/index.html`](static/index.html) is the page, with plain JavaScript and no build step. +- [`test_chat_app.py`](test_chat_app.py) tests the agent on the scripted model and the server's + Stream code, on the memory Store and on Redis. + +Run the tests from the repo root. + +```sh +uv run pytest tests/contrib/streams/examples/chat_app +``` diff --git a/tests/contrib/streams/examples/chat_app/__init__.py b/tests/contrib/streams/examples/chat_app/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/contrib/streams/examples/chat_app/activities.py b/tests/contrib/streams/examples/chat_app/activities.py new file mode 100644 index 0000000000..e8ce325a34 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/activities.py @@ -0,0 +1,65 @@ +"""The model Activity, which streams one round's token deltas to a tokens Stream.""" + +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncIterator + +from temporalio import activity +from temporalio.client import BufferedPublisher +from temporalio.exceptions import ApplicationError + +from .models import Model, RawContent +from .shared import ModelInput, ModelReply, Token, ToolCall + +# How many tokens the first attempt streams before the demo toggle fails it. +FAIL_AFTER = 8 + + +@contextlib.asynccontextmanager +async def heartbeats(every: float = 2) -> AsyncIterator[None]: + # A model can think for a while before its first delta, so the beat doesn't wait for them. + async def beat() -> None: + while True: + activity.heartbeat() + await asyncio.sleep(every) + + task = asyncio.create_task(beat()) + try: + yield + finally: + task.cancel() + + +class ModelActivities: + def __init__(self, model: Model) -> None: + self._model = model + + @activity.defn + async def call_model(self, input: ModelInput) -> ModelReply: + # A Stream from activity.client() writes as this attempt, so a retry withdraws its Records. + tokens = activity.client().to_stream(input.tokens) + fail = input.fail_first_attempt and activity.info().attempt == 1 + reply = ModelReply("") + text: list[str] = [] + async with heartbeats(), BufferedPublisher(tokens) as publisher: + streamed = self._model.stream(input.messages, input.tools) + async with contextlib.aclosing(streamed) as events: + async for event in events: + if isinstance(event, ToolCall): + reply.tool_calls.append(event) + elif isinstance(event, RawContent): + reply.raw = event.blocks + else: + publisher.publish(Token(input.number, event, input.round)) + text.append(event) + if fail and len(text) == FAIL_AFTER: + break + if fail: + # The block end would write the tokens too. The flush shows the call, and that + # the attempt fails only after readers have its tokens. + await publisher.flush() + raise ApplicationError("The fail toggle failed this attempt") + reply.text = "".join(text) + return reply diff --git a/tests/contrib/streams/examples/chat_app/docs.py b/tests/contrib/streams/examples/chat_app/docs.py new file mode 100644 index 0000000000..23510a3a05 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/docs.py @@ -0,0 +1,67 @@ +"""A small text about Streams, which the ``search_docs`` tool searches.""" + +DOCS = [ + ( + "Streams", + "A Stream is an ordered log of Records. Workflows, Activities and clients write to it, " + "and clients and Activities read it. Each Stream has an id that its creator picks.", + ), + ( + "Records and offsets", + "Each Record has an offset. Offsets are ordered and unique, with gaps. A reader saves " + "the offset after the last Record that it showed, and a later read resumes from there.", + ), + ( + "Producers", + "Each Record names its Producer, the writer that wrote it. A Workflow writes through " + "Local Activities, an Activity writes as its attempt, and a client writes as itself. The " + "Records of one Producer keep the order of its writes. Records of two Producers " + "interleave in the order that the Store gets them, and every reader sees that order.", + ), + ( + "Superseded and withdrawal", + "When an Activity attempt fails and its retry writes, a Superseded item withdraws the " + "Records of the failed attempt. A live reader crosses them out with withdrawn_offsets. " + "A reader that kept Records from an earlier read checks them with withdraws().", + ), + ( + "Retention", + "A Record stays readable for the Stream's Retention, also while the Stream is open. A " + "closed Stream is kept for one Retention after its close. Token deltas matter only to " + "live readers, so a chat gives its tokens Stream a short Retention, such as one hour.", + ), + ( + "Close", + "A run's end doesn't close a Stream. Code closes it with close() or at the end of an " + "async with block. Readers get the Records before the close, then their read ends.", + ), + ( + "Workflow Streams", + "A Workflow creates standalone Streams under ids of its choice. Each run of a chain " + "sends the same create, so a retry run gets the Stream that an earlier run made. A " + "Workflow can create a Stream at run time, such as a sub-agent that names its Streams " + "after its own Workflow ID.", + ), + ( + "Continue-as-new", + "A run that continues as new forwards the refs of its Streams to the next run. The next " + "run gets the Streams with to_stream and creates nothing, so readers see one Stream.", + ), + ( + "Reading", + "A read starts at an offset, at END for new Records only, or with the last N Records. " + "A read can wait for a Stream that doesn't exist yet with create_wait.", + ), +] + + +def search(query: str, count: int = 2) -> list[tuple[str, str]]: + """The sections that share the most words with the query.""" + words = {w.strip(".,?!").lower() for w in query.split() if len(w) > 3} + + def score(section: tuple[str, str]) -> int: + text = f"{section[0]} {section[1]}".lower() + return sum(1 for w in words if w in text) + + ranked = sorted(DOCS, key=score, reverse=True) + return [s for s in ranked[:count] if score(s) > 0] diff --git a/tests/contrib/streams/examples/chat_app/models/__init__.py b/tests/contrib/streams/examples/chat_app/models/__init__.py new file mode 100644 index 0000000000..ba97351317 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/models/__init__.py @@ -0,0 +1,69 @@ +"""The model providers. Each one streams one model round for a history and a set of tools.""" + +from __future__ import annotations + +import os +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from typing import Any, Protocol + +from ..shared import Message, ToolCall + +SYSTEM_PROMPT = ( + "You are the assistant in a demo of Temporal Streams. Use the tools when they help, and " + "keep each answer to one short paragraph." +) + + +@dataclass +class RawContent: + """A provider's own blocks of the round, which go back to it in the next round's history.""" + + blocks: list[dict[str, Any]] + + +RoundEvent = str | ToolCall | RawContent + + +class Model(Protocol): + provider: str + name: str + + def stream( + self, messages: list[Message], tools: list[str] + ) -> AsyncGenerator[RoundEvent, None]: + """Yield the round's text deltas, then its tool calls, then optionally its raw blocks. + + A refused or failed round raises ``ApplicationError``. + """ + ... + + +def pick_model() -> Model: + """The provider of ``CHAT_MODEL_PROVIDER``, else the first one whose key is set. + + This code checks only whether a key variable is set. Each SDK reads its key itself. + """ + provider = os.environ.get("CHAT_MODEL_PROVIDER") or ( + "claude" + if "ANTHROPIC_API_KEY" in os.environ + else "openai" + if "OPENAI_API_KEY" in os.environ + else "fake" + ) + # Each provider imports its SDK, so a demo with the fake model needs neither SDK. + if provider == "claude": + from .claude import ClaudeModel + + return ClaudeModel(os.environ.get("ANTHROPIC_MODEL", "claude-opus-5-5")) + if provider == "openai": + from .openai_chat import OpenAIModel + + return OpenAIModel(os.environ.get("OPENAI_MODEL", "gpt-4o-mini")) + if provider == "fake": + from .fake import FakeModel + + return FakeModel() + raise ValueError( + f"CHAT_MODEL_PROVIDER is {provider!r}, and it takes claude, openai or fake" + ) diff --git a/tests/contrib/streams/examples/chat_app/models/claude.py b/tests/contrib/streams/examples/chat_app/models/claude.py new file mode 100644 index 0000000000..a06ef3bce0 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/models/claude.py @@ -0,0 +1,153 @@ +"""Claude through the Messages API, one streamed round with tool use. + +The Workflow runs the agent loop, since each round and each tool is an Activity there, so this +module makes one request per round. +""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from typing import Any, cast + +# The SDK's dev environment has anthropic only on some Python versions. +import anthropic # type: ignore[import-not-found] + +from temporalio.exceptions import ApplicationError + +from ..shared import Message, ToolCall +from ..tools import TOOL_SPECS +from . import SYSTEM_PROMPT, RawContent, RoundEvent + + +def _history(messages: list[Message]) -> list[dict[str, Any]]: + history: list[dict[str, Any]] = [] + for m in messages: + if m.role == "user": + history.append({"role": "user", "content": m.text}) + elif m.role == "tool": + results = [ + { + "type": "tool_result", + "tool_use_id": r.call_id, + "content": r.text, + "is_error": r.error, + } + for r in m.results + ] + history.append({"role": "user", "content": results}) + elif m.raw is not None: + # A round's own blocks go back unchanged, so Claude keeps its thinking. + history.append({"role": "assistant", "content": m.raw}) + else: + # A round of another provider, such as before a restart with a new provider. + blocks: list[dict[str, Any]] = [{"type": "text", "text": m.text or "..."}] + blocks += [ + {"type": "tool_use", "id": c.id, "name": c.name, "input": c.args} + for c in m.tool_calls + ] + history.append({"role": "assistant", "content": blocks}) + return history + + +def _echo(content: list[Any]) -> list[dict[str, Any]]: + """The blocks of a round, as the next request sends them back.""" + fallback = max( + (i for i, b in enumerate(content) if b.type == "fallback"), default=-1 + ) + blocks: list[dict[str, Any]] = [] + for i, b in enumerate(content): + if b.type == "text": + blocks.append({"type": "text", "text": b.text}) + elif i < fallback: + # Before a fallback, only text goes back, since the fallback model can't read the + # declined model's other blocks. + continue + elif b.type == "thinking": + blocks.append( + {"type": "thinking", "thinking": b.thinking, "signature": b.signature} + ) + elif b.type == "redacted_thinking": + blocks.append({"type": "redacted_thinking", "data": b.data}) + elif b.type == "tool_use": + blocks.append( + {"type": "tool_use", "id": b.id, "name": b.name, "input": b.input} + ) + return blocks + + +class ClaudeModel: + """Streams a round from Claude. The client reads ``ANTHROPIC_API_KEY`` itself.""" + + provider = "claude" + + def __init__(self, name: str) -> None: + self.name = name + self._client = anthropic.AsyncAnthropic() + + async def stream( + self, messages: list[Message], tools: list[str] + ) -> AsyncGenerator[RoundEvent, None]: + specs = [ + { + "name": name, + "description": TOOL_SPECS[name]["description"], + "input_schema": TOOL_SPECS[name]["schema"], + # Inputs stream as they're made, and each tool checks its own input. + "eager_input_streaming": True, + } + for name in tools + ] + try: + async with self._client.beta.messages.stream( + model=self.name, + max_tokens=64000, + system=SYSTEM_PROMPT, + messages=cast(Any, _history(messages)), + tools=cast(Any, specs), + # A chat round needs little thinking, and this model can't turn thinking off. + output_config={"effort": "low"}, + # A declined request runs again on the model that Anthropic picks for its + # category, in the same call. + betas=["server-side-fallback-2026-07-01"], + fallbacks="default", + ) as stream: + async for text in stream.text_stream: + yield text + reply = await stream.get_final_message() + except anthropic.APIStatusError as err: + # An API error's text can quote the request, and History and the turns Stream keep + # the failure. So the failure has only the error's class, and no cause. + raise ApplicationError( + f"The Claude call failed with {type(err).__name__}", + type=type(err).__name__, + non_retryable=err.status_code < 500 and err.status_code != 429, + ) from None + except anthropic.APIConnectionError as err: + raise ApplicationError( + f"The Claude call failed with {type(err).__name__}", + type=type(err).__name__, + ) from None + except ValueError: + # A streamed tool input that isn't JSON at all, so the attempt asks again. + raise ApplicationError( + "Claude sent a tool input that isn't JSON", type="InvalidToolInput" + ) from None + if reply.stop_reason == "refusal": + # The whole fallback chain declined, so the text so far isn't an answer, and no + # tool of the round runs. + raise ApplicationError( + "Claude declined to answer this message", + type="Refusal", + non_retryable=True, + ) + calls = [b for b in reply.content if b.type == "tool_use"] + if calls and reply.stop_reason == "max_tokens": + # A cut-off tool input can still look complete, so no tool of the round runs. + raise ApplicationError( + "Claude's round was cut off in a tool call", + type="MaxTokens", + non_retryable=True, + ) + for b in calls: + yield ToolCall(b.id, b.name, b.input if isinstance(b.input, dict) else {}) + yield RawContent(_echo(list(reply.content))) diff --git a/tests/contrib/streams/examples/chat_app/models/fake.py b/tests/contrib/streams/examples/chat_app/models/fake.py new file mode 100644 index 0000000000..d873977864 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/models/fake.py @@ -0,0 +1,131 @@ +"""A scripted model, so the demo and the tests run with no API key. + +It plans each round from the turn's first message and the tool results so far, so the scenario +prompts take the same rounds as with a real model. +""" + +from __future__ import annotations + +import asyncio +import re +from collections.abc import AsyncGenerator +from typing import Any + +from ..shared import ROUNDS_PER_RUN, Message, ToolCall, ToolResult +from . import RoundEvent + +CANNED_REPLIES = [ + "I'm the scripted model, so this demo runs with no API key. Each word that you see is a " + "Record of the tokens Stream. The model Activity writes them through a BufferedPublisher, " + "which sends them in small batches.", + "The Workflow publishes each turn to the turns Stream, and the Activity publishes the tokens " + "to a Stream of their own. The browser reads both through the server, which sends each read " + "as Server-Sent Events.", + "Turn on the fail toggle and send a message. The first attempt fails partway through, and " + "the retry's Superseded item withdraws the tokens of the failed attempt. Then the round " + "streams again.", + "Reload the page. The tab keeps its offset in each Stream, and the server resumes each read " + "from it. A second tab is a second reader, and it reads the tokens from the end of the " + "Stream.", + f"After {ROUNDS_PER_RUN} model rounds, the Workflow continues as new. It forwards the refs of " + "its Streams to the next run, so the chat keeps its Streams.", +] + +Plan = tuple[str, list[tuple[str, dict[str, Any]]]] + + +def _turn(messages: list[Message]) -> list[Message]: + # The turn starts after the last answer, an assistant message that calls no tool. + start = 0 + for i, m in enumerate(messages[:-1]): + if m.role == "assistant" and not m.tool_calls: + start = i + 1 + return messages[start:] + + +def _numbers(text: str) -> list[int]: + return [int(n) for n in re.findall(r"\d+", text)] + + +def plan(messages: list[Message], tools: list[str]) -> Plan: + """The round's text and tool calls.""" + turn = _turn(messages) + prompt = turn[0].text + low = prompt.lower() + results: list[ToolResult] = [r for m in turn for r in m.results] + last = messages[-1] + if last.role == "user" and last is not turn[0] and "search_docs" in tools: + # A message that joined during the run gets a round of its own. + return ( + f'You also asked "{last.text}". I\'ll search the docs for that too.', + [("search_docs", {"query": last.text})], + ) + if low.startswith("research "): + topic = prompt[len("research ") :] + if not results: + return f"I'll search the docs for {topic}.", [ + ("search_docs", {"query": topic}) + ] + return f"Here is what the docs say about {topic}. {results[-1].text}", [] + if "research" in low and "research" in tools: + if not results: + topic = "continue-as-new" if "continue" in low else prompt + return "I'll ask a sub-agent to research that.", [ + ("research", {"topic": topic}) + ] + return f"The sub-agent reports back. {results[-1].text}", [] + if "six" in low and ("dice" in low or "die" in low): + rolls = [_numbers(r.text)[0] for r in results if r.name == "roll_dice"] + if len(rolls) < 6: + return f"Roll number {len(rolls) + 1}.", [("roll_dice", {"count": 1})] + return f"The six rolls are {rolls}, and they add up to {sum(rolls)}.", [] + if "dice" in low: + if not results: + return "I'll roll two dice first.", [("roll_dice", {"count": 2})] + if len(results) == 1: + a, b = _numbers(results[0].text)[:2] + return "Now I'll add them up.", [ + ("calculator", {"expression": f"{a} + {b}"}) + ] + return f"The dice {results[0].text}, and {results[1].text}.", [] + if "clock" in low or "time" in low: + if not results: + expression = ( + " ".join(re.findall(r"\d+\s*[*+/-]\s*\d+", prompt)[:1]) or "12 * 34" + ) + return "I'll ask the clock and the calculator at the same time.", [ + ("clock", {}), + ("calculator", {"expression": expression}), + ] + by_name = {r.name: r.text for r in results} + return f"It's {by_name['clock']}, and {by_name['calculator']}.", [] + if "search" in low or "docs" in low: + if not results: + return "I'll search the docs.", [("search_docs", {"query": prompt})] + found = " ".join(r.text.split("\n")[0] for r in results) + return f"Here is what I found. {found}", [] + asked = sum(1 for m in messages if m.role == "user") + return ( + f'You said "{prompt}". {CANNED_REPLIES[(asked - 1) % len(CANNED_REPLIES)]}', + [], + ) + + +class FakeModel: + """Streams each planned round word by word.""" + + provider = "fake" + name = "scripted" + + def __init__(self, delay: float = 0.05) -> None: + self._delay = delay + + async def stream( + self, messages: list[Message], tools: list[str] + ) -> AsyncGenerator[RoundEvent, None]: + text, calls = plan(messages, tools) + for word in text.split(" "): + await asyncio.sleep(self._delay) + yield word + " " + for i, (name, args) in enumerate(calls): + yield ToolCall(f"call_{len(messages)}_{i}", name, args) diff --git a/tests/contrib/streams/examples/chat_app/models/openai_chat.py b/tests/contrib/streams/examples/chat_app/models/openai_chat.py new file mode 100644 index 0000000000..af6f0eb54a --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/models/openai_chat.py @@ -0,0 +1,106 @@ +"""OpenAI chat completions, one streamed round with tool calls.""" + +from __future__ import annotations + +import json +from collections.abc import AsyncGenerator +from typing import Any, cast + +import openai + +from temporalio.exceptions import ApplicationError + +from ..shared import Message, ToolCall +from ..tools import TOOL_SPECS +from . import SYSTEM_PROMPT, RoundEvent + + +def _chat(messages: list[Message]) -> list[dict[str, Any]]: + chat: list[dict[str, Any]] = [{"role": "system", "content": SYSTEM_PROMPT}] + for m in messages: + if m.role == "user": + chat.append({"role": "user", "content": m.text}) + elif m.role == "tool": + chat += [ + {"role": "tool", "tool_call_id": r.call_id, "content": r.text} + for r in m.results + ] + else: + message: dict[str, Any] = {"role": "assistant", "content": m.text or None} + if m.tool_calls: + message["tool_calls"] = [ + { + "id": c.id, + "type": "function", + "function": {"name": c.name, "arguments": json.dumps(c.args)}, + } + for c in m.tool_calls + ] + chat.append(message) + return chat + + +class OpenAIModel: + """Streams a round from the OpenAI API. The client reads ``OPENAI_API_KEY`` itself.""" + + provider = "openai" + + def __init__(self, name: str) -> None: + self.name = name + self._client = openai.AsyncOpenAI() + + async def stream( + self, messages: list[Message], tools: list[str] + ) -> AsyncGenerator[RoundEvent, None]: + specs = [ + { + "type": "function", + "function": { + "name": name, + "description": TOOL_SPECS[name]["description"], + "parameters": TOOL_SPECS[name]["schema"], + }, + } + for name in tools + ] + # A tool call streams in parts, keyed by its index in the round. + calls: dict[int, dict[str, str]] = {} + try: + response = await self._client.chat.completions.create( + model=self.name, + messages=cast(Any, _chat(messages)), + tools=cast(Any, specs), + stream=True, + ) + async for chunk in response: + if not chunk.choices: + continue + delta = chunk.choices[0].delta + if delta.content: + yield delta.content + for part in delta.tool_calls or []: + call = calls.setdefault( + part.index, {"id": "", "name": "", "args": ""} + ) + call["id"] = part.id or call["id"] + if part.function: + call["name"] += part.function.name or "" + call["args"] += part.function.arguments or "" + except openai.APIError as err: + # An API error's text can quote the key, and History and the turns Stream keep the + # failure. So the failure has only the error's class, and no cause. + raise ApplicationError( + f"The OpenAI call failed with {type(err).__name__}", + type=type(err).__name__, + non_retryable=isinstance(err, openai.AuthenticationError), + ) from None + for index in sorted(calls): + call = calls[index] + try: + args = json.loads(call["args"] or "{}") + except ValueError: + # The tool checks its input and reports what's missing. + args = {} + yield ToolCall( + call["id"], call["name"], args if isinstance(args, dict) else {} + ) diff --git a/tests/contrib/streams/examples/chat_app/pyproject.toml b/tests/contrib/streams/examples/chat_app/pyproject.toml new file mode 100644 index 0000000000..af7389d392 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/pyproject.toml @@ -0,0 +1,19 @@ +# The chat app's own environment. It takes the SDK from this repo, so the build needs +# MATURIN_PEP517_ARGS="--features streams" for the Stream runtime. See the README. +[project] +name = "streams-chat-app" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "temporalio", + "fastapi>=0.135", + "uvicorn>=0.30", + "openai>=1.40", + "anthropic>=1.3", +] + +[tool.uv] +package = false + +[tool.uv.sources] +temporalio = { path = "../../../../..", editable = true } diff --git a/tests/contrib/streams/examples/chat_app/server.py b/tests/contrib/streams/examples/chat_app/server.py new file mode 100644 index 0000000000..1f985958b3 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/server.py @@ -0,0 +1,493 @@ +"""The chat server. It starts or opens chats, and serves their Streams as Server-Sent Events. + +It runs the sessions' Worker in its own process, so the in-memory backend works with no Redis. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import json +import os +import re +import uuid +from collections.abc import AsyncIterable, AsyncIterator, Sequence +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Annotated, Any + +from fastapi import Depends, FastAPI, HTTPException, Request +from fastapi.responses import FileResponse +from fastapi.sse import EventSourceResponse, ServerSentEvent +from pydantic import BaseModel + +from temporalio import streams +from temporalio.api.common.v1 import Payload +from temporalio.client import ( + Client, + Stream, + WorkflowExecutionStatus, + WorkflowFailureError, + WorkflowHandle, + WorkflowUpdateFailedError, +) +from temporalio.common import WorkflowIDReusePolicy +from temporalio.contrib.stream import RedisStreamBackend +from temporalio.contrib.stream.testing import MemoryStreamBackend +from temporalio.converter import DataConverter +from temporalio.exceptions import WorkflowAlreadyStartedError +from temporalio.service import RPCError, RPCStatusCode +from temporalio.worker import Worker + +from .activities import ModelActivities +from .models import pick_model +from .shared import ( + SessionInfo, + SessionInput, + SessionStreams, + SharedTurn, + SourceProducer, + Token, + Turn, + UserMessage, +) +from .tools import ToolActivities +from .workflow import ChatSession, Researcher + +LOBBY_ID = "streams-chat-app-lobby" +INDEX = Path(__file__).parent / "static" / "index.html" +SESSION_ID = r"[a-z0-9][a-z0-9-]{0,39}" +SHARE_ID = rf"({SESSION_ID})-snapshot-(\d+)" +# An older version of the app wrote snapshots of plain Turns under these ids. +OLD_SHARE_ID = rf"{SESSION_ID}-share-\d+" +# A sub-agent's Stream ids, which it picks from its Workflow ID and its run id. +AGENT_STREAM_ID = rf"{SESSION_ID}-agent-\d+-\d+-\d+-[0-9a-f]{{8}}-(turns|tokens)" +PAYLOADS = DataConverter.default.payload_converter +PRODUCER_METADATA = PAYLOADS.to_payload(streams.Producer(id="x")).metadata + + +@dataclasses.dataclass +class Chat: + client: Client + lobby: streams.StreamRef[SessionInfo] + task_queue: str + backend: str + provider: str + model: str + + +@contextlib.asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + redis_url = os.environ.get("CHAT_APP_REDIS_URL") + backend: streams.StreamBackend + if redis_url: + backend = RedisStreamBackend(redis_url, default=True) + else: + backend = MemoryStreamBackend(default=True) + client = await Client.connect( + os.environ.get("TEMPORAL_ADDRESS", "localhost:7233"), + stream_backends=[backend], + ) + # Each server start sends this create, so it gets the lobby that an earlier start made. + lobby = await client.create_or_get_stream[SessionInfo](LOBBY_ID) + model = pick_model() + task_queue = "streams-chat-app" + async with Worker( + client, + task_queue=task_queue, + workflows=[ChatSession, Researcher], + activities=[ModelActivities(model).call_model, ToolActivities().run_tool], + ): + app.state.chat = Chat( + client, + lobby.ref, + task_queue, + "redis" if redis_url else "memory", + model.provider, + model.name, + ) + yield + + +app = FastAPI(lifespan=lifespan) + + +def chat_of(request: Request) -> Chat: + return request.app.state.chat + + +ChatDep = Annotated[Chat, Depends(chat_of)] + + +def session_of(session_id: str, chat: ChatDep) -> WorkflowHandle[ChatSession, None]: + if not re.fullmatch(SESSION_ID, session_id): + raise HTTPException(400, "A chat name has lowercase letters, digits and dashes") + return chat.client.get_workflow_handle_for(ChatSession.run, session_id) + + +SessionDep = Annotated[WorkflowHandle[ChatSession, None], Depends(session_of)] + + +async def streams_of(session: SessionDep) -> SessionStreams: + try: + return await session.query(ChatSession.streams) + except RPCError as err: + if err.status == RPCStatusCode.NOT_FOUND: + raise HTTPException(404, f"There is no chat {session.id}") from err + raise + + +StreamsDep = Annotated[SessionStreams, Depends(streams_of)] + + +def snapshot_of(share_id: str, chat: ChatDep) -> Stream[SharedTurn]: + # A share link names a Stream id, so the server reads only the ids that it writes. + if not re.fullmatch(SHARE_ID, share_id): + raise HTTPException(404, f"There is no snapshot {share_id}") + return chat.client.get_stream[SharedTurn](share_id) + + +SnapshotDep = Annotated[Stream[SharedTurn], Depends(snapshot_of)] + + +@app.get("/") +async def index() -> FileResponse: + return FileResponse(INDEX) + + +@app.get("/share/{share_id}") +async def share_page() -> FileResponse: + return FileResponse(INDEX) + + +class OpenRequest(BaseModel): + session_id: str | None = None + + +@app.post("/api/sessions") +async def open_session(body: OpenRequest, chat: ChatDep) -> dict[str, Any]: + session = session_of(body.session_id or f"chat-{uuid.uuid4().hex[:6]}", chat) + try: + # An ended chat keeps its id, so its page opens it to read. + await chat.client.start_workflow( + ChatSession.run, + SessionInput(session.id), + id=session.id, + task_queue=chat.task_queue, + id_reuse_policy=WorkflowIDReusePolicy.REJECT_DUPLICATE, + ) + started = datetime.now(timezone.utc).isoformat(timespec="seconds") + await chat.client.to_stream(chat.lobby).publish( + SessionInfo(session.id, started) + ) + except WorkflowAlreadyStartedError: + pass + refs = await session.query(ChatSession.streams) + status = (await session.describe()).status + if status == WorkflowExecutionStatus.FAILED: + raise HTTPException(409, await failure_of(session)) + return { + "session_id": session.id, + "ended": status != WorkflowExecutionStatus.RUNNING, + "turns": refs.turns.id, + "tokens": refs.tokens.id, + "backend": chat.backend, + "provider": chat.provider, + "model": chat.model, + } + + +async def failure_of(session: WorkflowHandle[ChatSession, None]) -> str: + try: + await session.result() + except WorkflowFailureError as err: + return f"The chat {session.id} failed with {err.cause}" + return f"The chat {session.id} failed" + + +class SendRequest(BaseModel): + text: str + fail_first_attempt: bool = False + + +@app.post("/api/sessions/{session_id}/messages") +async def send(body: SendRequest, session: SessionDep) -> dict[str, Any]: + try: + offset = await session.execute_update( + ChatSession.send, UserMessage(body.text, body.fail_first_attempt) + ) + except WorkflowUpdateFailedError as err: + raise HTTPException(409, str(err.cause)) from err + except RPCError as err: + if err.status == RPCStatusCode.NOT_FOUND: + raise HTTPException(409, f"The chat {session.id} has ended") from err + raise + return {"offset": str(offset)} + + +@app.post("/api/sessions/{session_id}/add") +async def add(body: SendRequest, session: SessionDep) -> dict[str, Any]: + try: + await session.signal(ChatSession.add_message, UserMessage(body.text)) + except RPCError as err: + if err.status == RPCStatusCode.NOT_FOUND: + raise HTTPException(409, f"The chat {session.id} has ended") from err + raise + return {} + + +@app.post("/api/sessions/{session_id}/end") +async def end(session: SessionDep) -> dict[str, Any]: + try: + await session.signal(ChatSession.end) + except RPCError as err: + if err.status == RPCStatusCode.NOT_FOUND: + raise HTTPException(409, f"The chat {session.id} has ended") from err + raise + return {} + + +@app.post("/api/sessions/{session_id}/share") +async def share(session_id: str, refs: StreamsDep, chat: ChatDep) -> dict[str, Any]: + turns = chat.client.to_stream(refs.turns, payload_type=Turn) + # The snapshot holds the chat up to this offset, which names it. + upto = await turns.get_offset() + share_id = f"{session_id}-snapshot-{upto}" + shown: dict[int, SharedTurn] = {} + async with contextlib.aclosing(turns.subscribe(wait_new_records=False)) as read: + async for item in read: + if item.offset >= upto: + break + match item.kind: + case streams.ItemKind.SUPERSEDED: + for offset in item.withdrawn_offsets: + del shown[offset] + case streams.ItemKind.RECORD: + shown[item.offset] = SharedTurn( + item.data, item.offset, source_producer(item.producer) + ) + try: + # A strict create, since a share id names one snapshot. + async with await chat.client.create_stream[SharedTurn]( + share_id, retention=timedelta(days=1) + ) as snapshot: + await snapshot.publish(*shown.values()) + except streams.StreamClosedError: + pass # This point of the chat has a snapshot, and its block closed it. + except streams.StreamAlreadyExistsError: + pass # Another request writes this snapshot now, and readers wait for its close. + return {"share_id": share_id, "url": f"/share/{share_id}"} + + +@app.get("/api/shares/{share_id}") +async def share_info(share_id: str, chat: ChatDep) -> dict[str, Any]: + if re.fullmatch(OLD_SHARE_ID, share_id): + # Its Records have no origin and don't decode as SharedTurn, so the page asks for a share. + return {"share_id": share_id, "outdated": True} + snapshot = snapshot_of(share_id, chat) + try: + next_offset = await snapshot.get_offset() + except streams.StreamNotFoundError as err: + raise HTTPException(404, f"There is no snapshot {share_id}") from err + # The id names the chat and the offset that the snapshot goes up to. + named = re.fullmatch(SHARE_ID, share_id) + assert named is not None + session_id, upto = named.groups() + return { + "share_id": share_id, + "session_id": session_id, + "upto": upto, + "next_offset": str(next_offset), + } + + +@app.get("/api/shares/{share_id}/events", response_class=EventSourceResponse) +async def share_events(snapshot: SnapshotDep) -> AsyncIterable[ServerSentEvent]: + # A snapshot is closed, so the read ends after its last Turn. + async for event in events("turns", snapshot.subscribe()): + yield event + + +@app.get("/api/agents/events", response_class=EventSourceResponse) +async def agent_events( + turns: str, tokens: str, chat: ChatDep +) -> AsyncIterable[ServerSentEvent]: + # A page names a sub-agent's Streams by the ids in its Turn, so the server reads them by id. + if not ( + re.fullmatch(AGENT_STREAM_ID, turns) and re.fullmatch(AGENT_STREAM_ID, tokens) + ): + raise HTTPException(404, "There is no such sub-agent") + turns_read = chat.client.get_stream[Turn](turns).subscribe() + tokens_read = chat.client.get_stream[Token](tokens).subscribe() + merged = merge(events("sub-turns", turns_read), events("sub-tokens", tokens_read)) + async for event in merged: + yield event + + +@app.get("/api/lobby", response_class=EventSourceResponse) +async def lobby_events(chat: ChatDep) -> AsyncIterable[ServerSentEvent]: + async for event in events("lobby", lobby_read(chat)): + yield event + + +# A browser opens at most six connections to a host, so a tab reads all its Streams over one. +@app.get("/api/sessions/{session_id}/events", response_class=EventSourceResponse) +async def session_events( + refs: StreamsDep, + chat: ChatDep, + turns: str | None = None, + tokens: str | None = None, + kept: str | None = None, +) -> AsyncIterable[ServerSentEvent]: + # The tab sends the offsets that it saved, so a reload or a reconnect resumes there. The + # init's create can still be in flight when a new chat's tab starts to read. + wait = timedelta(seconds=30) + turns_stream = chat.client.to_stream(refs.turns, payload_type=Turn) + turns_read = turns_stream.subscribe( + offset=int(turns) if turns else None, create_wait=wait + ) + tokens_stream = chat.client.to_stream(refs.tokens, payload_type=Token) + # A tab that joins reads only new tokens, since the turns Stream has the earlier replies. + tokens_read = tokens_stream.subscribe( + offset=int(tokens) if tokens else streams.END, create_wait=wait + ) + merged = merge( + events("lobby", lobby_read(chat)), + events("turns", turns_read), + events("tokens", tokens_read, kept_producers(kept)), + ) + async for event in merged: + yield event + + +def lobby_read(chat: Chat) -> streams.Subscription[SessionInfo]: + lobby = chat.client.to_stream(chat.lobby, payload_type=SessionInfo) + # The newest 20 chats, then each new one. + return lobby.subscribe(last=20) + + +async def merge( + *sources: AsyncIterator[ServerSentEvent], +) -> AsyncIterator[ServerSentEvent]: + # A bound makes a fast read wait for the connection, as a read of its own would. + queue: asyncio.Queue[ServerSentEvent] = asyncio.Queue(maxsize=100) + + async def pump(source: AsyncIterator[ServerSentEvent]) -> None: + async for event in source: + await queue.put(event) + + tasks = [asyncio.create_task(pump(source)) for source in sources] + try: + while True: + yield await queue.get() + finally: + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +async def events( + name: str, + read: streams.Subscription[Any], + kept: Sequence[streams.Producer] = (), +) -> AsyncIterator[ServerSentEvent]: + try: + async for item in read: + data = {"stream": name, **item_json(item, kept)} + # Where a later read of this Stream resumes, which the tab saves. + data["next_offset"] = str(read.next_offset) if read.next_offset else None + yield ServerSentEvent(event="item", data=data) + yield ServerSentEvent(event="end", data={"stream": name}) + except Exception as err: + failure = {"stream": name, "error": type(err).__name__, "message": str(err)} + yield ServerSentEvent(event="failed", data=failure) + + +def item_json( + item: streams.Record[Any] | streams.Superseded, kept: Sequence[streams.Producer] +) -> dict[str, Any]: + # Offsets are past the range of a JavaScript number, so they go as strings. + match item.kind: + case streams.ItemKind.SUPERSEDED: + writer = item.facts.writer + return { + "kind": "superseded", + "offset": str(item.offset), + "producer": producer_json(writer.producer if writer else None), + "withdrawn_offsets": [str(o) for o in item.withdrawn_offsets], + # A tab sends the Producers of the Records that it kept from before a reload. + "withdrawn_kept": [i for i, p in enumerate(kept) if item.withdraws(p)], + } + case streams.ItemKind.RECORD: + return { + "kind": "record", + "offset": str(item.offset), + "producer": producer_json(item.producer), + "data": data_json(item.data), + } + + +def plain(value: Any) -> Any: + """A value with each ref in it as JSON, such as the refs of a sub-agent in a Turn.""" + if isinstance(value, streams.StreamRef): + return {"id": value.id, "backend": value.backend} + if isinstance(value, dict): + return {key: plain(item) for key, item in value.items()} + return value + + +def data_json(value: Any) -> dict[str, Any]: + data = plain(dataclasses.asdict(value)) + if isinstance(value, SharedTurn): + data["source_offset"] = str(value.source_offset) + return data + + +def producer_json(producer: streams.Producer | None) -> dict[str, Any] | None: + if producer is None: + return None + kind, attempt = epoch_of(producer.epoch) + return { + "id": producer.id, + "kind": kind, + "attempt": attempt, + # The tab sends it back after a reload, so the server can check its kept Records. + "wire": json.loads(PAYLOADS.to_payload(producer).data), + } + + +def source_producer(producer: streams.Producer | None) -> SourceProducer | None: + if producer is None: + return None + kind, attempt = epoch_of(producer.epoch) + return SourceProducer(producer.id, kind, attempt) + + +def epoch_of(epoch: streams.ProducerEpoch | None) -> tuple[str, int | None]: + match epoch.kind if epoch else None: + case streams.ProducerEpochKindWorkflow(): + return "workflow", None + case streams.ProducerEpochKindLocalActivity(value=local): + return "local activity", local.attempt + case ( + streams.ProducerEpochKindActivity(value=attempt) + | streams.ProducerEpochKindStandaloneActivity(value=attempt) + ): + return "activity", attempt.attempt + case streams.ProducerEpochKindClient(value=client): + return "client", client.attempt + case _: + return "unknown", None + + +def kept_producers(kept: str | None) -> list[streams.Producer]: + if not kept: + return [] + return [ + PAYLOADS.from_payload( + Payload(metadata=PRODUCER_METADATA, data=json.dumps(wire).encode()), + streams.Producer, + ) + for wire in json.loads(kept) + ] diff --git a/tests/contrib/streams/examples/chat_app/shared.py b/tests/contrib/streams/examples/chat_app/shared.py new file mode 100644 index 0000000000..f6a5c060d0 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/shared.py @@ -0,0 +1,191 @@ +"""The values that the chat Workflows, the Activities and the server pass around.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Literal + +from temporalio import streams + +# Low, so that the demo shows a continue-as-new after a few answers. +ROUNDS_PER_RUN = 6 + +# A model that keeps calling tools stops here, so a turn always ends. +MAX_ROUNDS = 8 + + +@dataclass +class SubAgent: + """The refs of a sub-agent's own Streams, which a Turn carries as data.""" + + id: str + turns: streams.StreamRef[Turn] + tokens: streams.StreamRef[Token] + + +@dataclass +class Turn: + """A Record of a turns Stream. + + ``number`` is the user turn that it belongs to, and ``round`` the model round in that turn. + """ + + role: Literal[ + "system", + "user", + "assistant", + "tool_call", + "tool_progress", + "tool_result", + "agent", + "error", + ] + text: str + number: int + round: int = 0 + # The round's text is the answer, and the turn is done. + final: bool = False + tool: str | None = None + call_id: str | None = None + agent: SubAgent | None = None + + +@dataclass +class Token: + """A Record of a tokens Stream, one model delta of a round.""" + + number: int + text: str + round: int = 1 + + +@dataclass +class ToolCall: + id: str + name: str + args: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class ToolResult: + call_id: str + name: str + text: str + error: bool = False + + +@dataclass +class Message: + """One message of a model's history.""" + + role: Literal["user", "assistant", "tool"] + text: str = "" + tool_calls: list[ToolCall] = field(default_factory=list) + results: list[ToolResult] = field(default_factory=list) + # A provider's own blocks of an assistant message, such as Claude's thinking blocks, which go + # back to the provider unchanged. + raw: list[dict[str, Any]] | None = None + + +@dataclass +class ModelReply: + text: str + tool_calls: list[ToolCall] = field(default_factory=list) + raw: list[dict[str, Any]] | None = None + + +@dataclass +class UserMessage: + text: str + # The demo toggle. The model Activity fails its first attempt partway through the round. + fail_first_attempt: bool = False + + +@dataclass +class Waiting: + number: int + message: UserMessage + + +@dataclass +class SessionStreams: + """The refs of a session's Streams. They are data, so a query hands them out.""" + + turns: streams.StreamRef[Turn] + tokens: streams.StreamRef[Token] + + +@dataclass +class SessionInput: + session_id: str + history: list[Message] = field(default_factory=list) + # The run before continue-as-new sets the rest. + streams: SessionStreams | None = None + run: int = 1 + number: int = 0 + waiting: list[Waiting] = field(default_factory=list) + + +@dataclass +class ModelInput: + messages: list[Message] + number: int + round: int + tokens: streams.StreamRef[Token] + tools: list[str] + fail_first_attempt: bool = False + + +@dataclass +class ToolInput: + call: ToolCall + number: int + round: int + turns: streams.StreamRef[Turn] + + +@dataclass +class ResearchInput: + topic: str + call: ToolCall + number: int + round: int + # The parent's turns Stream, where the sub-agent writes its link and its result. + parent_turns: streams.StreamRef[Turn] + + +@dataclass +class SourceProducer: + """The Producer of a copied Record in its first Stream.""" + + id: str + kind: str + attempt: int | None = None + + +@dataclass +class SharedTurn: + """A Record of a snapshot, a copy of a Turn and where the Turn was in the chat's turns Stream. + + The snapshot is a new Stream that the server writes, so its own offsets and Producer differ. + """ + + turn: Turn + source_offset: int + source_producer: SourceProducer | None = None + + +@dataclass +class SessionInfo: + """A Record of the lobby Stream, which lists the chats.""" + + session_id: str + started: str + + +def turns_id(session_id: str) -> str: + return f"{session_id}-turns" + + +def tokens_id(session_id: str) -> str: + return f"{session_id}-tokens" diff --git a/tests/contrib/streams/examples/chat_app/static/index.html b/tests/contrib/streams/examples/chat_app/static/index.html new file mode 100644 index 0000000000..00b0dc23ed --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/static/index.html @@ -0,0 +1,1262 @@ + + + + + +Streams chat + + + + +
+
+
Temporal Streams chat
+ + + + + +
+
+ + + +
+
+ + + +
+ + +
+ + +
+ +
+
+

Stream log

+
+
+ + not open + +
+
+ + not open + +
+
+ + not open + its own Streams +
+
+ + not open + streams-chat-app-lobby +
+
+
+
+
+ local activity the Workflow's writes + activity #n a model or tool attempt + client the server + superseded withdraws a failed attempt +
+
+
+ + + + diff --git a/tests/contrib/streams/examples/chat_app/test_chat_app.py b/tests/contrib/streams/examples/chat_app/test_chat_app.py new file mode 100644 index 0000000000..2d2c461ed2 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/test_chat_app.py @@ -0,0 +1,339 @@ +"""Runs the chat session on the scripted model, on the memory Store and on Redis.""" + +import json +import uuid + +import httpx + +from temporalio import streams +from temporalio.client import Client, WorkflowHandle +from temporalio.worker import Worker +from tests.contrib.streams.examples.chat_app import server +from tests.contrib.streams.examples.chat_app.activities import ( + FAIL_AFTER, + ModelActivities, +) +from tests.contrib.streams.examples.chat_app.models.fake import FakeModel +from tests.contrib.streams.examples.chat_app.shared import ( + ROUNDS_PER_RUN, + SessionInfo, + SessionInput, + SessionStreams, + SharedTurn, + Token, + Turn, + UserMessage, +) +from tests.contrib.streams.examples.chat_app.tools import ToolActivities +from tests.contrib.streams.examples.chat_app.workflow import ChatSession, Researcher +from tests.helpers import new_worker + + +def chat_worker(client: Client, work_scale: float = 0) -> Worker: + return new_worker( + client, + ChatSession, + Researcher, + activities=[ + ModelActivities(FakeModel(delay=0)).call_model, + ToolActivities(work_scale).run_tool, + ], + ) + + +async def start(client: Client, worker: Worker) -> WorkflowHandle[ChatSession, None]: + session_id = f"chat-{uuid.uuid4()}" + return await client.start_workflow( + ChatSession.run, + SessionInput(session_id), + id=session_id, + task_queue=worker.task_queue, + ) + + +async def read_until_answer( + read: streams.Subscription[Turn], number: int +) -> list[streams.Record[Turn]]: + records = [] + async for item in read: + if isinstance(item, streams.Record): + records.append(item) + if item.data.number == number and item.data.final: + break + return records + + +async def answer( + client: Client, worker: Worker, text: str +) -> tuple[WorkflowHandle[ChatSession, None], list[streams.Record[Turn]]]: + """Starts a session, sends one message, and reads its turns up to the answer.""" + handle = await start(client, worker) + refs = await handle.query(ChatSession.streams) + await handle.execute_update(ChatSession.send, UserMessage(text)) + read = client.to_stream(refs.turns, payload_type=Turn).subscribe() + records = await read_until_answer(read, 1) + await read.aclose() + return handle, records + + +async def test_turn_and_its_tokens(stream_client: Client): + async with chat_worker(stream_client) as worker: + handle = await start(stream_client, worker) + refs = await handle.query(ChatSession.streams) + offset = await handle.execute_update(ChatSession.send, UserMessage("hello")) + await handle.signal(ChatSession.end) + await handle.result() + # The session closed both Streams, so each read ends. + turns = stream_client.to_stream(refs.turns, payload_type=Turn) + records = [r async for r in turns.subscribe() if isinstance(r, streams.Record)] + tokens = stream_client.to_stream(refs.tokens, payload_type=Token) + deltas = [r.data async for r in tokens.subscribe() if isinstance(r, streams.Record)] + system, user, assistant, ended = records + assert system.data == Turn("system", "Run 1 of the session started", 0) + assert user.data == Turn("user", "hello", 1) and user.offset == offset + assert assistant.data.text.startswith('You said "hello".') + assert (assistant.data.round, assistant.data.final) == (1, True) + assert "".join(d.text for d in deltas) == assistant.data.text + assert {(d.number, d.round) for d in deltas} == {(1, 1)} + assert ended.data == Turn("system", "The chat ended", 1) + + +async def test_tool_rounds(stream_client: Client): + async with chat_worker(stream_client) as worker: + handle, records = await answer( + stream_client, + worker, + "Roll two dice, then add them up with the calculator.", + ) + await handle.signal(ChatSession.end) + turns = [r.data for r in records[1:]] + assert [(t.role, t.round, t.tool) for t in turns] == [ + ("user", 0, None), + ("assistant", 1, None), + ("tool_call", 1, "roll_dice"), + ("tool_progress", 1, "roll_dice"), + ("tool_result", 1, "roll_dice"), + ("assistant", 2, None), + ("tool_call", 2, "calculator"), + ("tool_progress", 2, "calculator"), + ("tool_result", 2, "calculator"), + ("assistant", 3, None), + ] + assert turns[-1].final and not any(t.final for t in turns[:-1]) + + +async def test_parallel_tools_interleave_their_turns(stream_client: Client): + async with chat_worker(stream_client, work_scale=1) as worker: + handle, records = await answer( + stream_client, worker, "What time is it, and what is 12 * 34?" + ) + await handle.signal(ChatSession.end) + calls = [r.data.tool for r in records if r.data.role == "tool_call"] + assert calls == ["clock", "calculator"] + # Each tool Activity is its own Producer. Its Turns keep their order, and the two tools' + # Turns interleave. + written = [r for r in records if r.data.role in ("tool_progress", "tool_result")] + assert [(r.data.tool, r.data.role) for r in written] == [ + ("clock", "tool_progress"), + ("calculator", "tool_progress"), + ("clock", "tool_result"), + ("calculator", "tool_result"), + ] or [(r.data.tool, r.data.role) for r in written] == [ + ("calculator", "tool_progress"), + ("clock", "tool_progress"), + ("clock", "tool_result"), + ("calculator", "tool_result"), + ] + assert len({r.producer.id for r in written if r.producer}) == 2 + assert records[-1].data.text.endswith("12 * 34 = 408. ") + + +async def test_message_during_a_run_joins_the_next_round(stream_client: Client): + async with chat_worker(stream_client, work_scale=1) as worker: + handle = await start(stream_client, worker) + refs = await handle.query(ChatSession.streams) + await handle.execute_update( + ChatSession.send, UserMessage("Search the docs for retention.") + ) + read = stream_client.to_stream(refs.turns, payload_type=Turn).subscribe() + records = [] + async for item in read: + if isinstance(item, streams.Record): + records.append(item.data) + if item.data.role == "tool_call": + # The search runs for a while, and the user writes during it. + await handle.signal( + ChatSession.add_message, UserMessage("How long?") + ) + break + records += [r.data for r in await read_until_answer(read, 1)] + await read.aclose() + await handle.signal(ChatSession.end) + assert Turn("user", "How long?", 1) in records + searches = [t.text for t in records if t.role == "tool_call"] + assert searches == [ + "search_docs(Search the docs for retention.)", + "search_docs(How long?)", + ] + assert records[-1].final and records[-1].round == 3 + + +async def test_sub_agent_writes_its_own_streams(stream_client: Client): + async with chat_worker(stream_client) as worker: + handle, records = await answer( + stream_client, worker, "Use the research tool to look into continue-as-new." + ) + await handle.signal(ChatSession.end) + roles = [r.data.role for r in records[1:]] + assert roles == [ + "user", + "assistant", + "tool_call", + "agent", + "tool_result", + "assistant", + ] + [link] = [r.data for r in records if r.data.role == "agent"] + assert link.agent is not None + # The sub-agent closed its Streams, so each read ends. + child = stream_client.to_stream(link.agent.turns, payload_type=Turn) + child_turns = [ + r.data async for r in child.subscribe() if isinstance(r, streams.Record) + ] + assert [(t.role, t.tool) for t in child_turns] == [ + ("user", None), + ("assistant", None), + ("tool_call", "search_docs"), + ("tool_progress", "search_docs"), + ("tool_result", "search_docs"), + ("assistant", None), + ] + tokens = stream_client.to_stream(link.agent.tokens, payload_type=Token) + assert [r async for r in tokens.subscribe() if isinstance(r, streams.Record)] + assert records[-1].data.text.startswith("The sub-agent reports back.") + + +async def failed_attempt( + client: Client, +) -> tuple[SessionStreams, list[streams.Record[Token] | streams.Superseded]]: + """The refs of a session whose one round failed its first attempt, and the token items.""" + async with chat_worker(client) as worker: + handle = await start(client, worker) + refs = await handle.query(ChatSession.streams) + await handle.execute_update( + ChatSession.send, UserMessage("hello", fail_first_attempt=True) + ) + await handle.signal(ChatSession.end) + await handle.result() + tokens = client.to_stream(refs.tokens, payload_type=Token) + return refs, [item async for item in tokens.subscribe()] + + +async def test_retry_withdraws_the_failed_attempt(stream_client: Client): + _, items = await failed_attempt(stream_client) + [superseded] = [i for i in items if isinstance(i, streams.Superseded)] + records = [i for i in items if isinstance(i, streams.Record)] + withdrawn = [r for r in records if r.offset in superseded.withdrawn_offsets] + kept = [r for r in records if r.offset not in superseded.withdrawn_offsets] + assert len(withdrawn) == FAIL_AFTER + assert all(superseded.withdraws(r) for r in withdrawn) + assert not any(superseded.withdraws(r) for r in kept) + # The retry streamed the whole round again. + assert "".join(r.data.text for r in kept).startswith( + "".join(r.data.text for r in withdrawn) + ) + + +async def test_server_checks_kept_tokens_with_withdraws(stream_client: Client): + refs, items = await failed_attempt(stream_client) + shown = [i for i in items if isinstance(i, streams.Record)][:FAIL_AFTER] + # A tab that reloaded after it showed the failed attempt's tokens sends their Producer, and + # its read resumes after them. + producer = server.producer_json(shown[0].producer) + assert producer is not None + kept = server.kept_producers(json.dumps([producer["wire"]])) + tokens = stream_client.to_stream(refs.tokens, payload_type=Token) + [superseded] = [ + server.item_json(item, kept) + async for item in tokens.subscribe(offset=shown[-1].offset + 1) + if isinstance(item, streams.Superseded) + ] + assert superseded["withdrawn_offsets"] == [] + assert superseded["withdrawn_kept"] == [0] + + +async def test_resume_from_a_saved_offset(stream_client: Client): + async with chat_worker(stream_client) as worker: + handle = await start(stream_client, worker) + refs = await handle.query(ChatSession.streams) + turns = stream_client.to_stream(refs.turns, payload_type=Turn) + await handle.execute_update(ChatSession.send, UserMessage("message 1")) + read = turns.subscribe() + await read_until_answer(read, 1) + saved = read.next_offset + await read.aclose() + # Each of these answers takes one round, so the last one comes from the run after + # continue-as-new. + last = ROUNDS_PER_RUN + 1 + for number in range(2, last + 1): + await handle.execute_update( + ChatSession.send, UserMessage(f"message {number}") + ) + resumed = turns.subscribe(offset=saved) + later = [r.data for r in await read_until_answer(resumed, last)] + await resumed.aclose() + assert await handle.query(ChatSession.streams) == refs + await handle.signal(ChatSession.end) + await handle.result() + assert later[0] == Turn("user", "message 2", 2) + assert [t.number for t in later if t.final] == list(range(2, last + 1)) + assert any(t.text == "Run 2 of the session started" for t in later) + + +async def test_share_again_is_a_strict_create_of_an_existing_id(stream_client: Client): + async with chat_worker(stream_client) as worker: + lobby = await stream_client.create_stream[SessionInfo](f"lobby-{uuid.uuid4()}") + server.app.state.chat = server.Chat( + stream_client, lobby.ref, worker.task_queue, "test", "fake", "scripted" + ) + transport = httpx.ASGITransport(app=server.app) + async with httpx.AsyncClient( + transport=transport, base_url="http://app" + ) as http: + session_id = (await http.post("/api/sessions", json={})).json()[ + "session_id" + ] + await http.post(f"/api/sessions/{session_id}/messages", json={"text": "hi"}) + handle = stream_client.get_workflow_handle(session_id) + refs = await handle.query(ChatSession.streams) + turns = stream_client.to_stream(refs.turns, payload_type=Turn) + read = turns.subscribe() + await read_until_answer(read, 1) + await read.aclose() + chat = [ + r + async for r in turns.subscribe(wait_new_records=False) + if isinstance(r, streams.Record) + ] + first = (await http.post(f"/api/sessions/{session_id}/share")).json() + # The same point of the chat gives the same id, whose snapshot is closed. + again = (await http.post(f"/api/sessions/{session_id}/share")).json() + info = await http.get(f"/api/shares/{first['share_id']}") + missing = await http.get(f"/api/shares/{session_id}-snapshot-1") + outdated = await http.get(f"/api/shares/{session_id}-share-1") + await http.post(f"/api/sessions/{session_id}/end") + await handle.result() + assert again == first + assert info.status_code == 200 and missing.status_code == 404 + assert info.json()["session_id"] == session_id + assert outdated.json()["outdated"] is True + # The read ends at the snapshot's close. Each copy names its Turn's offset in the chat. + snapshot = stream_client.get_stream[SharedTurn](first["share_id"]) + copies = [ + r.data async for r in snapshot.subscribe() if isinstance(r, streams.Record) + ] + assert [c.turn for c in copies] == [r.data for r in chat] + assert [c.source_offset for c in copies] == [r.offset for r in chat] + assert {c.source_producer.kind for c in copies if c.source_producer} == { + "local activity" + } diff --git a/tests/contrib/streams/examples/chat_app/tools.py b/tests/contrib/streams/examples/chat_app/tools.py new file mode 100644 index 0000000000..d5175b6a81 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/tools.py @@ -0,0 +1,170 @@ +"""The agent's tools. Each one is local and makes no network call. + +``research`` starts a sub-agent, so the Workflow runs it as a child Workflow. The others run in +the ``run_tool`` Activity, which writes the tool's Turns to the chat's turns Stream itself. +""" + +from __future__ import annotations + +import ast +import asyncio +import operator +import random +from datetime import datetime, timezone +from typing import Any + +from temporalio import activity + +from . import docs +from .shared import ToolInput, ToolResult, Turn + +TOOL_SPECS: dict[str, dict[str, Any]] = { + "calculator": { + "description": "Work out an arithmetic expression, such as '12 * 34'. Call it for any " + "sum or product instead of working it out yourself.", + "schema": { + "type": "object", + "properties": {"expression": {"type": "string"}}, + "required": ["expression"], + }, + }, + "clock": { + "description": "Get the current time in UTC. Call it when the user asks for the time.", + "schema": {"type": "object", "properties": {}}, + }, + "roll_dice": { + "description": "Roll dice and get each value. Call it when the user asks for a roll.", + "schema": { + "type": "object", + "properties": { + "count": {"type": "integer", "minimum": 1, "maximum": 6}, + "sides": {"type": "integer", "minimum": 2, "maximum": 100}, + }, + "required": ["count"], + }, + }, + "search_docs": { + "description": "Search a short text about Temporal Streams. Call it for any question " + "about Streams, Records, offsets, Retention or withdrawal.", + "schema": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + "research": { + "description": "Start a sub-agent that researches a topic about Temporal Streams in the " + "docs, and get its summary. Call it when the user asks for research or a sub-agent.", + "schema": { + "type": "object", + "properties": {"topic": {"type": "string"}}, + "required": ["topic"], + }, + }, +} + +# How long each tool works, so that tools that run at the same time interleave their Turns. +WORK_SECONDS = {"calculator": 0.6, "clock": 0.3, "roll_dice": 0.4, "search_docs": 0.8} + +_OPERATORS: dict[type, Any] = { + ast.Add: operator.add, + ast.Sub: operator.sub, + ast.Mult: operator.mul, + ast.Div: operator.truediv, + ast.FloorDiv: operator.floordiv, + ast.Mod: operator.mod, + ast.Pow: operator.pow, + ast.USub: operator.neg, +} + + +def _evaluate(node: ast.AST) -> float: + # Only numbers and arithmetic, since the expression comes from a model. + if isinstance(node, ast.Constant) and isinstance(node.value, (int, float)): + return node.value + if isinstance(node, ast.UnaryOp) and type(node.op) in _OPERATORS: + return _OPERATORS[type(node.op)](_evaluate(node.operand)) + if isinstance(node, ast.BinOp) and type(node.op) in _OPERATORS: + left, right = _evaluate(node.left), _evaluate(node.right) + if isinstance(node.op, ast.Pow) and abs(right) > 10: + raise ValueError("the exponent is too large") + return _OPERATORS[type(node.op)](left, right) + raise ValueError("only numbers and + - * / // % ** are allowed") + + +def calculator(args: dict[str, Any]) -> str: + expression = args.get("expression") + if not isinstance(expression, str): + raise ValueError("expression must be a string") + value = _evaluate(ast.parse(expression, mode="eval").body) + return f"{expression} = {value:g}" + + +def clock(_args: dict[str, Any]) -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + + +def roll_dice(args: dict[str, Any]) -> str: + count, sides = args.get("count", 1), args.get("sides", 6) + if not isinstance(count, int) or not 1 <= count <= 6: + raise ValueError("count must be an integer from 1 to 6") + if not isinstance(sides, int) or not 2 <= sides <= 100: + raise ValueError("sides must be an integer from 2 to 100") + # Seeded by the call, so a retry of the attempt rolls the same values. + info = activity.info() + rolls = random.Random(f"{info.workflow_id}:{info.activity_id}").choices( + range(1, sides + 1), k=count + ) + return "rolled " + ", ".join(str(r) for r in rolls) + + +def search_docs(args: dict[str, Any]) -> str: + query = args.get("query") + if not isinstance(query, str): + raise ValueError("query must be a string") + found = docs.search(query) + if not found: + return "The docs have nothing on that." + return "\n".join(f"{title}. {text}" for title, text in found) + + +TOOLS = { + "calculator": calculator, + "clock": clock, + "roll_dice": roll_dice, + "search_docs": search_docs, +} + + +def describe(name: str, args: dict[str, Any]) -> str: + """A short form of a call, such as ``calculator(12 * 34)``.""" + return f"{name}({', '.join(str(v) for v in args.values())})" + + +class ToolActivities: + def __init__(self, work_scale: float = 1.0) -> None: + self._work_scale = work_scale + + @activity.defn + async def run_tool(self, input: ToolInput) -> ToolResult: + call = input.call + # A Stream from activity.client() writes as this attempt, so each tool is a Producer. + turns = activity.client().to_stream(input.turns) + + def turn(role: Any, text: str) -> Turn: + return Turn( + role, text, input.number, input.round, tool=call.name, call_id=call.id + ) + + await turns.publish(turn("tool_progress", f"{call.name} started")) + await asyncio.sleep(WORK_SECONDS.get(call.name, 0) * self._work_scale) + tool = TOOLS.get(call.name) + try: + if tool is None: + raise ValueError(f"there is no tool {call.name}") + # A streamed tool input can be cut off or malformed, so each tool checks its input. + result = ToolResult(call.id, call.name, tool(call.args)) + except (ValueError, SyntaxError, ZeroDivisionError) as err: + result = ToolResult(call.id, call.name, f"error: {err}", error=True) + await turns.publish(turn("tool_result", result.text)) + return result diff --git a/tests/contrib/streams/examples/chat_app/workflow.py b/tests/contrib/streams/examples/chat_app/workflow.py new file mode 100644 index 0000000000..e1e58602a7 --- /dev/null +++ b/tests/contrib/streams/examples/chat_app/workflow.py @@ -0,0 +1,333 @@ +"""The chat session Workflow, its agent loop and its sub-agent.""" + +from __future__ import annotations + +import asyncio +from datetime import timedelta + +from temporalio import workflow +from temporalio.common import RetryPolicy +from temporalio.exceptions import ActivityError, ApplicationError, ChildWorkflowError + +from .shared import ( + MAX_ROUNDS, + ROUNDS_PER_RUN, + Message, + ModelInput, + ResearchInput, + SessionInput, + SessionStreams, + SubAgent, + Token, + ToolCall, + ToolInput, + ToolResult, + Turn, + UserMessage, + Waiting, + tokens_id, + turns_id, +) + +with workflow.unsafe.imports_passed_through(): + from .activities import ModelActivities + from .tools import TOOL_SPECS, ToolActivities, describe + +# The pause before a retry keeps the withdrawn tokens on screen for a moment. +MODEL_RETRY = RetryPolicy( + initial_interval=timedelta(seconds=1), backoff_coefficient=1.0, maximum_attempts=3 +) + + +def notice(err: ActivityError | ChildWorkflowError) -> str: + cause = err.cause + return cause.message if isinstance(cause, ApplicationError) else str(cause) + + +class Agent: + """The model rounds of an answer, which a chat session and a sub-agent share. + + Each round streams its tokens. The turns Stream gets a Turn for each round, each tool call + and each tool result. + """ + + def __init__( + self, + turns: workflow.WorkflowStream[Turn], + tokens: workflow.WorkflowStream[Token], + tools: list[str], + history: list[Message], + ) -> None: + self.turns = turns + self.tokens = tokens + self.tools = tools + self.history = history + self.rounds = 0 + self.number = 0 + self.answering = False + # Messages that came during an answer, which join its next round. + self.added: list[UserMessage] = [] + + async def answer(self, number: int, fail_first_attempt: bool = False) -> str: + self.number, self.answering = number, True + for round in range(1, MAX_ROUNDS + 1): + self.rounds += 1 + self.history += [Message("user", m.text) for m in self.added] + self.added.clear() + try: + reply = await workflow.execute_activity_method( + ModelActivities.call_model, + ModelInput( + self.history, + number, + round, + self.tokens.ref, + self.tools, + fail_first_attempt and round == 1, + ), + start_to_close_timeout=timedelta(minutes=5), + # An attempt whose Worker stops misses its heartbeats, so its retry starts in + # seconds and withdraws its tokens. + heartbeat_timeout=timedelta(seconds=10), + retry_policy=MODEL_RETRY, + ) + except ActivityError as err: + return await self._stop(Turn("error", notice(err), number, round)) + self.history.append( + Message("assistant", reply.text, reply.tool_calls, raw=reply.raw) + ) + # A message that came during the round gets another round. + final = not reply.tool_calls and not self.added + if final: + # A message that comes from here on starts a new turn. + self.answering = False + await self.turns.publish( + Turn("assistant", reply.text, number, round, final=final) + ) + if final: + return reply.text + if reply.tool_calls: + await self.turns.publish( + *( + Turn( + "tool_call", + describe(c.name, c.args), + number, + round, + tool=c.name, + call_id=c.id, + ) + for c in reply.tool_calls + ) + ) + # The round's tools run at the same time, and each one writes its own Turns. + results = await asyncio.gather( + *( + self._call(c, number, round, i) + for i, c in enumerate(reply.tool_calls) + ) + ) + self.history.append(Message("tool", results=list(results))) + text = f"The agent stopped after {MAX_ROUNDS} rounds" + return await self._stop(Turn("error", text, number, MAX_ROUNDS)) + + async def _stop(self, turn: Turn) -> str: + self.answering = False + # The next turn's model still sees a message that came during this one. + self.history += [Message("user", m.text) for m in self.added] + self.added.clear() + await self.turns.publish(turn) + return "" + + async def _call( + self, call: ToolCall, number: int, round: int, index: int + ) -> ToolResult: + try: + if call.name == "research": + return await workflow.execute_child_workflow( + Researcher.run, + ResearchInput( + str(call.args.get("topic", "")), + call, + number, + round, + self.turns.ref, + ), + id=f"{workflow.info().workflow_id}-agent-{number}-{round}-{index}", + ) + return await workflow.execute_activity_method( + ToolActivities.run_tool, + ToolInput(call, number, round, self.turns.ref), + start_to_close_timeout=timedelta(minutes=1), + ) + except (ActivityError, ChildWorkflowError) as err: + # The tool didn't write its result, so the Workflow writes the failure. + result = ToolResult(call.id, call.name, f"error: {notice(err)}", error=True) + await self.turns.publish( + Turn( + "tool_result", + result.text, + number, + round, + tool=call.name, + call_id=call.id, + ) + ) + return result + + +@workflow.defn +class ChatSession: + @workflow.init + def __init__(self, input: SessionInput) -> None: + self._waiting = list(input.waiting) + self._number = input.number + self._ended = False + if input.streams is None: + # A chat id can come back, such as in a restarted session, so the run gets the + # chat's Streams if they exist. + turns = workflow.create_or_get_stream_nowait[Turn]( + turns_id(input.session_id) + ) + # Token deltas matter only to live readers, so they get a short Retention. + tokens = workflow.create_or_get_stream_nowait[Token]( + tokens_id(input.session_id), retention=timedelta(hours=1) + ) + else: + turns = workflow.to_stream(input.streams.turns) + tokens = workflow.to_stream(input.streams.tokens) + self._turns = turns + self._agent = Agent(turns, tokens, list(TOOL_SPECS), list(input.history)) + # The init can't await, so it publishes with the _nowait call. + turns.publish_nowait( + Turn("system", f"Run {input.run} of the session started", self._number) + ) + + @workflow.query + def streams(self) -> SessionStreams: + return SessionStreams(self._turns.ref, self._agent.tokens.ref) + + @workflow.update + async def send(self, message: UserMessage) -> int: + self._number += 1 + number = self._number + # The awaited publish gives the Record's offset, which the server hands to the browser. + offset = await self._turns.publish(Turn("user", message.text, number)) + self._waiting.append(Waiting(number, message)) + return offset + + @send.validator + def check_send(self, message: UserMessage) -> None: + if not message.text.strip(): + raise ValueError("The message is empty") + if self._ended: + raise ValueError("The chat has ended") + + @workflow.signal + def add_message(self, message: UserMessage) -> None: + """A message that the user sends while the agent works.""" + if self._ended or not message.text.strip(): + return + if self._agent.answering: + # It joins the next round of the answer in progress. + self._agent.added.append(message) + number = self._agent.number + else: + self._number += 1 + number = self._number + self._waiting.append(Waiting(number, message)) + # A signal handler can't await, so the user's Turn goes out with publish_nowait. + self._turns.publish_nowait(Turn("user", message.text, number)) + + @workflow.signal + def end(self) -> None: + self._ended = True + + @workflow.run + async def run(self, input: SessionInput) -> None: + agent = self._agent + while True: + # A send that started before the end still adds its message, so the end waits for it. + await workflow.wait_condition( + lambda: ( + bool(self._waiting) + or (self._ended and workflow.all_handlers_finished()) + ) + ) + if not self._waiting: + break + waiting = self._waiting.pop(0) + agent.history.append(Message("user", waiting.message.text)) + await agent.answer(waiting.number, waiting.message.fail_first_attempt) + if self._ended or ( + agent.rounds < ROUNDS_PER_RUN + and not workflow.info().is_continue_as_new_suggested() + ): + continue + await workflow.wait_condition(workflow.all_handlers_finished) + # The next run wouldn't know of an end that came during the wait. + if not self._ended: + # Forward the refs, so the next run keeps the Streams and creates nothing. + workflow.continue_as_new( + SessionInput( + session_id=input.session_id, + history=agent.history, + streams=self.streams(), + run=input.run + 1, + number=self._number, + waiting=self._waiting, + ) + ) + await self._turns.publish(Turn("system", "The chat ended", self._number)) + # A run's end doesn't close a Stream, so the session closes both. + await self._turns.close() + await agent.tokens.close() + + +@workflow.defn +class Researcher: + """A sub-agent, which answers one research question in Streams of its own.""" + + @workflow.run + async def run(self, input: ResearchInput) -> ToolResult: + info = workflow.info() + # The ids include this run's id, which the parent can't know. So the sub-agent creates + # its Streams itself, and hands their refs to the parent in a Turn. + own_id = f"{info.workflow_id}-{info.run_id[:8]}" + turns = await workflow.create_stream[Turn](turns_id(own_id)) + tokens = await workflow.create_stream[Token]( + tokens_id(own_id), retention=timedelta(hours=1) + ) + call = input.call + parent = workflow.to_stream(input.parent_turns) + await parent.publish( + Turn( + "agent", + f"A sub-agent researches {input.topic}", + input.number, + input.round, + tool=call.name, + call_id=call.id, + agent=SubAgent(own_id, turns.ref, tokens.ref), + ) + ) + # The block end closes both Streams, so a reader of the sub-agent sees it end. + async with turns, tokens: + prompt = f"Research {input.topic}" + await turns.publish(Turn("user", prompt, 1)) + agent = Agent(turns, tokens, ["search_docs"], [Message("user", prompt)]) + summary = await agent.answer(1) + result = ToolResult( + call.id, call.name, summary or "The sub-agent found nothing." + ) + await parent.publish( + Turn( + "tool_result", + result.text, + input.number, + input.round, + tool=call.name, + call_id=call.id, + ) + ) + return result From 4b1459be16dff3fdb037cad585af0d3426c57815 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sat, 7 Nov 2026 17:46:29 -0700 Subject: [PATCH 2/7] Thinned the Streams facade onto the regenerated client and the API-review decisions The facade delegates `create_stream`/`get_stream` to the regenerated nexgen client and matches the approved API. Renamed `publish`/`subscribe` to `append`/`read`, added the opaque `StreamOffset`, collapsed the errors onto reason-coded `StreamError`, and made `producer()` a module-level context manager. Dropped `create_or_get_stream` and `to_stream`, and moved Retention onto the backends. --- temporalio/bridge/streams.py | 8 +- temporalio/client/__init__.py | 6 +- temporalio/client/_client.py | 178 ++++++--- temporalio/client/_streams.py | 312 +++++---------- temporalio/streams/__init__.py | 52 ++- temporalio/streams/_access.py | 13 +- temporalio/streams/_activity.py | 12 +- temporalio/streams/_activity_stream.py | 39 +- temporalio/streams/_conflict.py | 26 ++ temporalio/streams/_errors.py | 112 +----- temporalio/streams/_items.py | 7 +- temporalio/streams/_offset.py | 91 +++++ temporalio/streams/_reader.py | 35 +- temporalio/streams/_ref.py | 20 +- temporalio/streams/_stream_client.py | 164 +++++--- temporalio/streams/_workflow.py | 100 ++--- temporalio/streams/_writer.py | 31 +- temporalio/streams/_writes.py | 5 +- .../streams/system/client/_extensions.py | 213 +++++----- .../system/client/_resources/stream.py | 16 +- temporalio/streams/system/models.py | 7 +- .../system/workflow/_resources/stream.py | 208 ++++++---- .../workflow/operations/create_stream.py | 107 ++--- temporalio/workflow/__init__.py | 8 +- temporalio/workflow/_streams.py | 301 ++++++--------- .../contrib/streams/test_activity_streams.py | 272 +++++++------ tests/contrib/streams/test_bindings.py | 16 +- tests/contrib/streams/test_facade.py | 100 ++--- tests/contrib/streams/test_pydantic.py | 14 +- tests/contrib/streams/test_reader.py | 18 +- tests/contrib/streams/test_redis.py | 171 ++++---- tests/contrib/streams/test_refs.py | 10 +- tests/contrib/streams/test_stream_client.py | 363 +++++++++-------- tests/contrib/streams/test_workflow_module.py | 365 ++++++++++-------- 34 files changed, 1801 insertions(+), 1599 deletions(-) create mode 100644 temporalio/streams/_conflict.py create mode 100644 temporalio/streams/_offset.py diff --git a/temporalio/bridge/streams.py b/temporalio/bridge/streams.py index 4b4a7f59dd..4af50df5f0 100644 --- a/temporalio/bridge/streams.py +++ b/temporalio/bridge/streams.py @@ -22,11 +22,15 @@ def __init__( """Build the runtime on Redis, or on memory with ``memory``, from its encoded config. Raises: - ValueError: The bridge build doesn't link the crate, or the config is not valid. + StreamConfigError: The bridge build doesn't link the crate. + ValueError: The config is not valid. """ new = getattr(temporalio.bridge.temporal_sdk_bridge, "new_stream_runtime", None) if new is None: - raise ValueError("This bridge build has no Stream runtime") + # Imported here, since temporalio.streams._errors imports this module. + from temporalio.streams._errors import StreamConfigError + + raise StreamConfigError("This bridge build has no Stream runtime") self._ref: Any = new(runtime._ref, memory, config) @property diff --git a/temporalio/client/__init__.py b/temporalio/client/__init__.py index 70ff482adf..40bc726c49 100644 --- a/temporalio/client/__init__.py +++ b/temporalio/client/__init__.py @@ -185,7 +185,7 @@ ScheduleUpdate, ScheduleUpdateInput, ) -from ._streams import BufferedPublisher, Stream, SyncStream, SyncSubscription +from ._streams import BufferedAppender, Stream, SyncReader, SyncStream from ._worker_versioning import ( BuildIdOp, BuildIdOpAddNewCompatible, @@ -398,8 +398,8 @@ "ParamType", "ReturnType", "SelfType", - "BufferedPublisher", + "BufferedAppender", "Stream", + "SyncReader", "SyncStream", - "SyncSubscription", ] diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index 158fa016fe..25997af2df 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -95,7 +95,7 @@ ScheduleBackfill, ScheduleHandle, ) -from ._streams import CreateStream, GetStream, Stream +from ._streams import Stream from ._worker_versioning import ( BuildIdOp, TaskReachabilityType, @@ -112,7 +112,12 @@ ) if TYPE_CHECKING: - from temporalio.streams import StreamBackend, StreamRef + from temporalio.streams import ( + Execution, + StreamBackend, + StreamIDConflictPolicy, + StreamRef, + ) from ._interceptor import Interceptor from ._plugin import Plugin @@ -916,79 +921,154 @@ def get_workflow_handle( result_type=result_type, ) - @property - def create_stream(self) -> CreateStream[Any]: - """Create a Stream, which fails if a Stream of the id exists. + @overload + async def create_stream( + self, + id: str, + /, + *, + payload_type: type[_StreamPayloadT], + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, + rpc_metadata: Mapping[str, str | bytes] = {}, + rpc_timeout: timedelta | None = None, + ) -> Stream[_StreamPayloadT]: ... - Give the payload type as a type argument:: + @overload + async def create_stream( + self, + id: str, + /, + *, + payload_type: None = None, + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, + rpc_metadata: Mapping[str, str | bytes] = {}, + rpc_timeout: timedelta | None = None, + ) -> Stream[Any]: ... - stream = await client.create_stream[Turn]("chat-42", retention=timedelta(days=1)) + async def create_stream( + self, + id: str, + /, + *, + payload_type: type[_StreamPayloadT] | None = None, + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, + rpc_metadata: Mapping[str, str | bytes] = {}, + rpc_timeout: timedelta | None = None, + ) -> Stream[Any]: + """Create a Stream, and return it. + + Give the payload type with ``payload_type=``:: + + stream = await client.create_stream("chat-42", payload_type=Turn) async with stream: # optional, the block end closes it - await stream.publish(Turn("user", "hi")) + await stream.append(Turn("user", "hi")) - A Stream must exist before its first publish, and the create calls the Store, so a bad + A Stream must exist before its first append, and the create calls the Store, so a bad setting fails early. Code closes the Stream with ``close()``, or at the end of - ``async with``. The call takes the Stream id, an optional ``backend=`` name and the - options of :py:class:`temporalio.streams.StreamOptions`. See - :py:meth:`CreateStream.__call__`. + ``async with``. - .. warning:: - This API is experimental. - """ - return CreateStream(self, None, or_get=False) - - @property - def create_or_get_stream(self) -> CreateStream[Any]: - """Create a Stream, or get the one of the id that exists. + Args: + id: The Stream id, which the caller picks, as for a Workflow ID. + payload_type: The type that reads decode each Record with. Defaults to no type, so + a read decodes each Record as a JSON value. + backend: The name of the backend. Defaults to the registered backend with + ``default=True``. + id_conflict_policy: ``FAIL`` (the default) raises + :py:class:`StreamAlreadyExistsError` on an id that exists. ``USE_EXISTING`` + returns the existing Stream. + rpc_metadata: Headers of the call. The Redis and memory backends ignore it. + rpc_timeout: The deadline of the call, on every backend. - It takes the arguments of :py:attr:`create_stream`. On a Stream that exists, it ignores - the options, so its retries are safe. It still fails on a closed or expired Stream. + Raises: + StreamAlreadyExistsError: ``FAIL`` on an id that exists. + StreamConfigError: The client has no backend with that name, or no backend has + ``default=True`` and the call gives no backend. No call is made. .. warning:: This API is experimental. """ - return CreateStream(self, None, or_get=True) + from temporalio.streams import StreamIDConflictPolicy, _stream_client - @property - def get_stream(self) -> GetStream[Any]: - """Get a Stream by its id, with no call. + return await _stream_client.create( + self, + payload_type, + id=id, + backend=backend, + id_conflict_policy=id_conflict_policy + if id_conflict_policy is not None + else StreamIDConflictPolicy.FAIL, + rpc_metadata=rpc_metadata, + rpc_timeout=rpc_timeout, + ) - Give the payload type as a type argument, as in ``client.get_stream[Turn]("chat-42")``. - Reads decode each Record with it, or as a JSON value with no type argument. For a ref - that came as data, such as from a Workflow's query, use :py:meth:`to_stream`. See - :py:meth:`GetStream.__call__`. + @overload + def get_stream( + self, ref: StreamRef[_StreamPayloadT], / + ) -> Stream[_StreamPayloadT]: ... - .. warning:: - This API is experimental. - """ - return GetStream(self, None) + @overload + def get_stream( + self, + id: str, + /, + *, + payload_type: type[_StreamPayloadT], + owner: Execution | None = None, + backend: str | None = None, + ) -> Stream[_StreamPayloadT]: ... - def to_stream( + @overload + def get_stream( self, - ref: StreamRef[_StreamPayloadT], + id: str, + /, *, - payload_type: type[_StreamPayloadT] | None = None, - ) -> Stream[_StreamPayloadT]: - """Get the Stream of a ref that came as data, such as from a query. It makes no call. + payload_type: None = None, + owner: Execution | None = None, + backend: str | None = None, + ) -> Stream[Any]: ... - The type argument of the ref is for type checkers only. Without ``payload_type``, a read - gives each Record as a JSON value, such as a dict for a dataclass. + def get_stream( + self, + id_or_ref: str | StreamRef[Any], + /, + *, + payload_type: type[Any] | None = None, + owner: Execution | None = None, + backend: str | None = None, + ) -> Stream[Any]: + """Get a Stream by its id, or by a ref that came as data. It makes no call. - .. warning:: - This API is experimental. + Give the payload type with ``payload_type=``, as in + ``client.get_stream("chat-42", payload_type=Turn)``. Reads decode each Record with it, or + as a JSON value with no type. Pass a :py:class:`StreamRef` that came as data, such as + from a Workflow's query, to get its Stream. Args: - ref: The ref of the Stream. - payload_type: The type that reads decode each Record with. Defaults to no type, so - a read decodes each Record as a JSON value. + id_or_ref: The Stream id, or a ref of the Stream. + payload_type: The type that reads decode each Record with, for the id form. + owner: The owner of a bound Stream, for the id form. + backend: The name of the backend, for the id form. Defaults to the registered + backend with ``default=True``. Raises: - StreamConfigError: The client has no backend of the ref, or the ref has no backend - and no backend has ``default=True``. + StreamConfigError: The client has no backend of the Stream, or none has + ``default=True`` for a Stream with no backend. + ValueError: The id is empty. + + .. warning:: + This API is experimental. """ from temporalio.streams import _stream_client - return _stream_client.to_stream(self, ref, payload_type=payload_type) + if isinstance(id_or_ref, str): + return _stream_client.get_stream( + self, payload_type, id=id_or_ref, owner=owner, backend=backend + ) + return _stream_client.from_ref(self, id_or_ref) def get_workflow_handle_for( self, diff --git a/temporalio/client/_streams.py b/temporalio/client/_streams.py index 03a07c0bf2..62b5373ba1 100644 --- a/temporalio/client/_streams.py +++ b/temporalio/client/_streams.py @@ -12,39 +12,35 @@ from types import TracebackType from typing import TYPE_CHECKING, Any, Generic, TypeVar -from typing_extensions import Self, Unpack +from typing_extensions import Self import temporalio.workflow if TYPE_CHECKING: import temporalio.api.stream.v1 - import temporalio.client from temporalio.streams import ( End, + Reader, Record, - StreamOptions, + StreamOffset, StreamRef, - Subscription, Superseded, ) from temporalio.streams._stream_client import StreamCalls T = TypeVar("T") -_U = TypeVar("_U") class Stream(Generic[T]): """A Stream outside Workflow code, with its calls. ``T`` is the payload type. - Get one with :py:attr:`temporalio.client.Client.create_stream`, - :py:attr:`temporalio.client.Client.create_or_get_stream`, - :py:attr:`temporalio.client.Client.get_stream`, or - :py:meth:`temporalio.client.Client.to_stream` for a ref that came as data. Its - :py:attr:`ref` is the data to pass on, such as to a Workflow. + Get one with :py:meth:`temporalio.client.Client.create_stream` or + :py:meth:`temporalio.client.Client.get_stream`. Its :py:attr:`ref` is the data to pass on, + such as to a Workflow. - All Streams of one client and Stream id write as one Producer, and :py:meth:`producer` - returns a Stream that writes as a Producer with a name of its own. A write error that no - caller got raises at the next call of the Stream. + All Streams of one client and Stream id write as one Producer, and + :py:func:`temporalio.streams.producer` sets a Producer with a name of its own for the writes + in a block. A write error that no caller got raises at the next call of the Stream. In an Activity, including a Local Activity, a Stream from the Worker's client, such as from ``activity.client()``, writes as the Producer of the attempt. So a retry withdraws the @@ -59,7 +55,7 @@ class Stream(Generic[T]): def __init__( self, ref: StreamRef[T], payload_type: Any, calls: StreamCalls ) -> None: - """Create a Stream. Use a client's ``create_stream``, ``get_stream`` or ``to_stream``.""" + """Create a Stream. Use a client's ``create_stream`` or ``get_stream``.""" self._ref = ref # The type that a read decodes each value with. None decodes with no type hint. self._payload_type = payload_type @@ -77,12 +73,12 @@ def proto_ref(self) -> temporalio.api.stream.v1.StreamRef: return _ref.to_proto(self._ref) - def publish( + def append( self, *values: T, rpc_metadata: Mapping[str, str | bytes] = {}, rpc_timeout: timedelta | None = None, - ) -> asyncio.Future[int]: + ) -> asyncio.Future[StreamOffset]: """Start the write of one Record for each value, in order, and return its future. The write has its place in the Stream's order when the call returns, also when no @@ -104,21 +100,20 @@ def publish( Raises: StreamClosedError: This object closed the Stream. - StreamUnsupportedError: The code is a sync Activity, which uses - :py:class:`SyncStream`. + StreamError: The code is a sync Activity, which uses :py:class:`SyncStream`. """ - return self._calls.publish(values, rpc_metadata, rpc_timeout) + return self._calls.append(values, rpc_metadata, rpc_timeout) - def subscribe( + def read( self, *, - offset: int | End | None = None, + offset: StreamOffset | End | None = None, last: int | None = None, wait_new_records: bool = True, create_wait: timedelta | None = None, rpc_metadata: Mapping[str, str | bytes] = {}, rpc_timeout: timedelta | None = None, - ) -> Subscription[T]: + ) -> Reader[T]: """Read the Stream's items in order. The read ends when the Stream is closed and the read gave its last item, or at the @@ -141,29 +136,32 @@ def subscribe( whole read. A call that waits for new Records returns before the deadline. Raises: - StreamUnsupportedError: The code is a sync Activity, which uses - :py:class:`SyncStream`. + StreamError: The code is a sync Activity, which uses :py:class:`SyncStream`. """ from temporalio.streams._access import read_spec + from temporalio.streams._items import End + wire_offset: int | End | None + if offset is None or isinstance(offset, End): + wire_offset = offset + else: + wire_offset = offset._wire read = read_spec( self._payload_type, - offset=offset, + offset=wire_offset, last=last, wait_new_records=wait_new_records, create_wait=create_wait, ) - return self._calls.subscribe(read, rpc_metadata, rpc_timeout) + return self._calls.read(read, rpc_metadata, rpc_timeout) async def get_offset( self, *, rpc_metadata: Mapping[str, str | bytes] = {}, rpc_timeout: timedelta | None = None, - ) -> int: - """The next offset of the Stream, the newest offset plus one, or 0 if it has no Record. - - Offsets are sparse and above zero, so ``offset=0`` reads from the start. + ) -> StreamOffset: + """The next offset of the Stream, past the newest Record, or the beginning if it's empty. Args: rpc_metadata: Headers of the call. The Redis and memory backends ignore it. @@ -177,9 +175,9 @@ async def close( rpc_metadata: Mapping[str, str | bytes] = {}, rpc_timeout: timedelta | None = None, ) -> None: - """Write the Records that this object published, then close the Stream. + """Write the Records that this object appended, then close the Stream. - A close carries no value, so code with a last value publishes it first. Readers get the + A close carries no value, so code with a last value appends it first. Readers get the Records before the close, then their read ends. Later writes fail with ``StreamClosedError``. @@ -188,9 +186,8 @@ async def close( rpc_timeout: The deadline of each call, on every backend. Raises: - StreamNotOwnerError: The code can't close the Stream. Only the run chain closes a - Stream bound to a Workflow, and an Activity closes no Stream. Both raise with no - call. + StreamError: The code can't close the Stream. Only the run chain closes a Stream + bound to a Workflow, and an Activity closes no Stream. Both raise with no call. """ await self._calls.close(rpc_metadata, rpc_timeout) @@ -198,7 +195,7 @@ async def __aenter__(self) -> Self: """Use the Stream in ``async with``, which closes it at the end. Raises: - StreamNotOwnerError: The code can't close the Stream. See :py:meth:`close`. + StreamError: The code can't close the Stream. See :py:meth:`close`. """ await self._calls.enter() return self @@ -215,54 +212,35 @@ async def __aexit__( """ await self._calls.exit(exc) - def producer(self, name: str, epoch: int) -> Stream[T]: - """Return this Stream as a Stream that writes as the Producer ``name`` at ``epoch``. - - Use it for a writer that must fence its earlier instances, such as a server that - restarts. Keep ``epoch`` in durable storage, and raise it at each start. After a write - at a higher epoch, a write at a lower one raises - :py:class:`temporalio.streams.StreamFencedError`. The Records of the earlier epochs - aren't withdrawn, since they're real messages. Run only one live writer for each name. - - Args: - name: The Producer name, the same across restarts. - epoch: The writer's epoch, 1 or more. - - Raises: - ValueError: The epoch is below 1. - StreamUnsupportedError: The Stream writes as an Activity attempt. - """ - return Stream(self._ref, self._payload_type, self._calls.producer(name, epoch)) - def __repr__(self) -> str: """Show the ref.""" return f"{type(self).__name__}({self._ref!r})" -class SyncSubscription(Generic[T]): +class SyncReader(Generic[T]): """The items of one read of a Stream in a sync Activity, in order. Iterate it with ``for``. - Each iteration blocks the Activity's thread. See :py:class:`temporalio.streams.Subscription`. + Each iteration blocks the Activity's thread. See :py:class:`temporalio.streams.Reader`. .. warning:: This API is experimental. """ - def __init__(self, subscription: Subscription[T]) -> None: - """Create a read. Use a ``SyncStream``'s ``subscribe`` instead.""" + def __init__(self, reader: Reader[T]) -> None: + """Create a read. Use a ``SyncStream``'s ``read`` instead.""" from temporalio.streams import _activity_stream - self._subscription = subscription + self._reader = reader self._steps: Generator[Record[T] | Superseded, None, None] = ( - _activity_stream.sync_items(subscription) + _activity_stream.sync_items(reader) ) @property - def next_offset(self) -> int | None: - """The offset to resume from. See :py:attr:`temporalio.streams.Subscription.next_offset`.""" - return self._subscription.next_offset + def next_offset(self) -> StreamOffset | None: + """The offset to resume from. See :py:attr:`temporalio.streams.Reader.next_offset`.""" + return self._reader.next_offset - def __iter__(self) -> SyncSubscription[T]: + def __iter__(self) -> SyncReader[T]: """The read is its own iterator.""" return self @@ -281,7 +259,7 @@ class SyncStream(Generic[T]): It wraps a :py:class:`Stream` of the Worker's client, such as for a ref of the Activity's input. A sync Activity has no ``activity.client()``, so it keeps the Worker's client, such as in an Activity class. Each call runs the Stream's call of the same name on the Worker's event - loop, and blocks until it ends. A publish returns the offset, not a future. + loop, and blocks until it ends. An append returns the offset, not a future. .. warning:: This API is experimental. @@ -291,7 +269,7 @@ def __init__(self, stream: Stream[T]) -> None: """Wrap ``stream`` for a sync Activity.""" self._stream = stream - def publish(self, *values: T) -> int: + def append(self, *values: T) -> StreamOffset: """Write one Record for each value, in order, and return the last Record's offset. A list is one value. @@ -301,37 +279,34 @@ def publish(self, *values: T) -> int: """ from temporalio.streams import _activity_stream - async def publish() -> int: - return await self._stream.publish(*values) + async def append() -> StreamOffset: + return await self._stream.append(*values) - return _activity_stream.run_blocking(publish()) + return _activity_stream.run_blocking(append()) - def subscribe( + def read( self, *, - offset: int | End | None = None, + offset: StreamOffset | End | None = None, last: int | None = None, wait_new_records: bool = True, create_wait: timedelta | None = None, - ) -> SyncSubscription[T]: - """Read the Stream's items in order. See :py:meth:`Stream.subscribe`.""" + ) -> SyncReader[T]: + """Read the Stream's items in order. See :py:meth:`Stream.read`.""" from temporalio.streams import _activity_stream - async def subscribe() -> Subscription[T]: - return self._stream.subscribe( + async def read() -> Reader[T]: + return self._stream.read( offset=offset, last=last, wait_new_records=wait_new_records, create_wait=create_wait, ) - return SyncSubscription(_activity_stream.run_blocking(subscribe())) + return SyncReader(_activity_stream.run_blocking(read())) - def get_offset(self) -> int: - """The next offset of the Stream, the newest offset plus one, or 0 if it has no Record. - - Offsets are sparse and above zero, so ``offset=0`` reads from the start. - """ + def get_offset(self) -> StreamOffset: + """The next offset of the Stream, past the newest Record, or the beginning if empty.""" from temporalio.streams import _activity_stream return _activity_stream.run_blocking(self._stream.get_offset()) @@ -340,7 +315,7 @@ def close(self) -> None: """Close the Stream. See :py:meth:`Stream.close`. Raises: - StreamNotOwnerError: The Stream writes as the Activity attempt, which can't close it. + StreamError: The Stream writes as the Activity attempt, which can't close it. """ from temporalio.streams import _activity_stream @@ -351,20 +326,20 @@ def close(self) -> None: _BATCH = 1000 -class BufferedPublisher(Generic[T]): - """Batches the publishes of one Stream. +class BufferedAppender(Generic[T]): + """Batches the appends of one Stream. - Use it in an Activity or a client that writes often. :py:meth:`publish` returns at once. - The publisher writes the buffered values as one batch at most ``max_delay`` after the first + Use it in an Activity or a client that writes often. :py:meth:`append` returns at once. + The appender writes the buffered values as one batch at most ``max_delay`` after the first of them, or at once when they fill a batch. One write is in flight at a time, and values - published during it wait for the next one. Use it in ``async with``, which writes the + appended during it wait for the next one. Use it in ``async with``, which writes the remaining values at the end of the block. For a Stream that writes as an Activity attempt, the end of the attempt also writes the remaining values. - ``await flush()`` is the backpressure point. A caller that publishes faster than the Store + ``await flush()`` is the backpressure point. A caller that appends faster than the Store accepts Records must await it from time to time, so the buffer doesn't grow without bound. - After a batch fails, the publisher writes nothing more, so a reader sees no gap. Each later + After a batch fails, the appender writes nothing more, so a reader sees no gap. Each later call raises the error. .. warning:: @@ -377,21 +352,21 @@ def __init__( *, max_delay: timedelta = timedelta(milliseconds=100), ) -> None: - """Create a publisher. + """Create an appender. Args: - stream: The Stream to publish to. + stream: The Stream to append to. max_delay: The longest time that a value waits in the buffer. Raises: - StreamUnsupportedError: The code is Workflow code. There, the publishes that wait - behind a write to the same Stream go as one Local Activity. + StreamError: The code is Workflow code. There, the appends that wait behind a write + to the same Stream go as one Local Activity. """ from temporalio.streams import _activity_stream, _errors if temporalio.workflow.in_workflow(): - raise _errors.StreamUnsupportedError( - "Workflow code can't use a BufferedPublisher. Its publishes that wait behind " + raise _errors.unsupported( + "Workflow code can't use a BufferedAppender. Its appends that wait behind " "a write already go as one Local Activity." ) self._stream = stream @@ -401,41 +376,41 @@ def __init__( self._first_at = 0.0 self._timer: asyncio.TimerHandle | None = None self._writing: asyncio.Task[None] | None = None - self._published = 0 + self._appended = 0 # Count of the values that a write took, written or not. self._settled = 0 self._error: BaseException | None = None - self._last_offset = 0 + self._last_offset: StreamOffset | None = None self._unwritten = 0 self._reported = False attempt = _activity_stream.attempt_of(stream) if attempt is not None: - attempt.publishers.append(self) + attempt.appenders.append(self) - def publish(self, *values: T) -> None: + def append(self, *values: T) -> None: """Add one Record for each value to the buffer, and return at once. A list is one value. Raises: - StreamError: A batch failed to write, so the publisher takes no more values. + StreamError: A batch failed to write, so the appender takes no more values. """ self._raise_error() if not values: - raise TypeError("publish takes one or more values") + raise TypeError("append takes one or more values") loop = asyncio.get_running_loop() if not self._buffer: self._first_at = loop.time() self._buffer.extend(values) - self._published += len(values) + self._appended += len(values) self._schedule() - async def flush(self) -> int: - """Write every value published before the call, and return the last written offset. + async def flush(self) -> StreamOffset | None: + """Write every value appended before the call, and return the last written offset. - Returns 0 if the publisher hasn't written anything. + Returns None if the appender hasn't written anything. """ - target = self._published + target = self._appended while self._settled < target: if self._writing is None: self._start() @@ -491,20 +466,20 @@ async def _write(self, batch: list[T]) -> None: finally: self._settled += len(batch) self._writing = None - # Schedule the values published during the write, for when their delay ends or they + # Schedule the values appended during the write, for when their delay ends or they # fill a batch. self._schedule() - async def _send(self, batch: list[T]) -> int: - return await self._stream.publish(*batch) + async def _send(self, batch: list[T]) -> StreamOffset: + return await self._stream.append(*batch) @property def unwritten(self) -> int: - """How many published values a failed batch left unwritten.""" + """How many appended values a failed batch left unwritten.""" return self._unwritten - async def __aenter__(self) -> BufferedPublisher[T]: - """Use the publisher in ``async with``.""" + async def __aenter__(self) -> BufferedAppender[T]: + """Use the appender in ``async with``.""" return self async def __aexit__( @@ -519,114 +494,3 @@ async def __aexit__( except Exception: if exc is None: raise - - -# Python has no type arguments on a call, so a call that takes the payload type is an object -# whose subscript binds the type, as in client.get_stream[Turn]("chat-42"). The subscript passes -# the class on, so a read decodes each Record with it, and type checkers see Stream[Turn]. With -# no subscript, T is Any and a read gives JSON values. - - -class CreateStream(Generic[T]): - """The call of :py:attr:`temporalio.client.Client.create_stream` and - :py:attr:`temporalio.client.Client.create_or_get_stream`. - - Subscript it with the payload type, as in ``client.create_stream[Turn]("chat-42")``. - - .. warning:: - This API is experimental. - """ - - def __init__( - self, - client: temporalio.client.Client, - payload_type: type[T] | None, - *, - or_get: bool, - ) -> None: - """Create the call. Use the client's property instead.""" - self._client = client - self._payload_type = payload_type - self._or_get = or_get - - def __getitem__(self, payload_type: type[_U]) -> CreateStream[_U]: - """The call for Streams of ``payload_type``. - - It takes a class, such as ``Turn`` or ``list[str]``. Type checkers refuse a union, such - as ``str | None``, so use ``Any`` or a class that holds the value instead. - """ - return CreateStream(self._client, payload_type, or_get=self._or_get) - - async def __call__( - self, - id: str, - *, - backend: str | None = None, - **options: Unpack[StreamOptions], - ) -> Stream[T]: - """Create the Stream, and return it. - - Args: - id: The Stream id, which the caller picks, as for a Workflow ID. - backend: The name of the backend. Defaults to the registered backend with - ``default=True``. - options: The options of a new Stream, such as ``retention``. See - :py:class:`temporalio.streams.StreamOptions`. - - Raises: - StreamAlreadyExistsError: ``create_stream`` of an id that exists. - StreamClosedError: A Stream of that id closed. - StreamExpiredError: A Stream of that id expired. - StreamConfigError: The client has no backend with that name, or no backend has - ``default=True`` and the call gives no backend. No call is made. - """ - from temporalio.streams import _stream_client - - return await _stream_client.create( - self._client, - self._payload_type, - id=id, - backend=backend, - options=options, - or_get=self._or_get, - ) - - -class GetStream(Generic[T]): - """The call of :py:attr:`temporalio.client.Client.get_stream`. - - Subscript it with the payload type, as in ``client.get_stream[Turn]("chat-42")``. - - .. warning:: - This API is experimental. - """ - - def __init__( - self, client: temporalio.client.Client, payload_type: type[T] | None - ) -> None: - """Create the call. Use the client's property instead.""" - self._client = client - self._payload_type = payload_type - - def __getitem__(self, payload_type: type[_U]) -> GetStream[_U]: - """The call for Streams of ``payload_type``. It takes a class, as for ``create_stream``.""" - return GetStream(self._client, payload_type) - - def __call__(self, id: str, *, backend: str | None = None) -> Stream[T]: - """Get the Stream with no call. - - Args: - id: The Stream id. - backend: The name of the backend. Defaults to the registered backend with - ``default=True``. - - Raises: - StreamConfigError: The client has no backend with that name, or no backend has - ``default=True`` and the call gives no backend. - ValueError: The id is empty. - """ - from temporalio.streams import _stream_client - - return _stream_client.get_stream( - self._client, self._payload_type, id=id, backend=backend - ) diff --git a/temporalio/streams/__init__.py b/temporalio/streams/__init__.py index aacd02b6ca..845f6ea451 100644 --- a/temporalio/streams/__init__.py +++ b/temporalio/streams/__init__.py @@ -1,26 +1,27 @@ """The Streams API, which Workflows, Activities and clients share. A Stream is an ordered log of Records that Workflows, Activities and clients write, and that -clients and Activities read. A Stream has one payload type, and a call takes it as a type -argument, as in ``client.get_stream[Turn]("chat-42")``. With no type argument, the payload type -is ``Any``, and a read gives JSON values. +clients and Activities read. A Stream has one payload type, and a call takes it as the +``payload_type`` argument, as in ``client.get_stream("chat-42", payload_type=Turn)``. With no +payload type, a read gives JSON values. -Code creates a Stream before its first publish. :py:data:`temporalio.workflow.create_stream` and -:py:attr:`temporalio.client.Client.create_stream` fail with :py:class:`StreamAlreadyExistsError` if -the id exists, and ``create_or_get_stream`` gets the Stream instead. In Workflow code, each run of a -chain sends the same create for an id, so a retry run or a cron run gets the Stream that an earlier -run made. Workflow code gets a :py:class:`temporalio.workflow.WorkflowStream`, and other code a +Code creates a Stream before its first append. :py:data:`temporalio.workflow.create_stream` and +:py:meth:`temporalio.client.Client.create_stream` fail with :py:class:`StreamAlreadyExistsError` +if the id exists. Pass ``id_conflict_policy=StreamIDConflictPolicy.USE_EXISTING` to get the +Stream instead. In Workflow code, each run of a chain sends the same create for an id, so a retry +run or a cron run gets the Stream that an earlier run made. Workflow code gets a +:py:class:`temporalio.workflow.WorkflowStream`, and other code a :py:class:`temporalio.client.Stream`. A client gets any Stream by its id with -:py:attr:`temporalio.client.Client.get_stream`, with no call. +:py:meth:`temporalio.client.Client.get_stream`, with no call. A Stream's :py:class:`StreamRef` is data with no calls. Code passes it as an argument to a Workflow, an Activity, a child Workflow or the next run, or hands it out through a query. -:py:func:`temporalio.workflow.to_stream` and :py:meth:`temporalio.client.Client.to_stream` give +:py:func:`temporalio.workflow.get_stream` and :py:meth:`temporalio.client.Client.get_stream` give the Stream of a ref. In an Activity, a Stream from ``activity.client()`` writes as the attempt. A read gives :py:class:`Record` and :py:class:`Superseded` items, and the ``kind`` of each item -narrows its type. A close carries no value. A read of a closed Stream ends once it gives the last -item. +narrows its type. A :py:class:`StreamOffset` is the opaque position of a Record. A close carries +no value. A read of a closed Stream ends once it gives the last item. A :py:class:`StreamBackend` makes the Stream calls and registers a backend name. Examples are :py:class:`temporalio.contrib.stream.RedisStreamBackend` (``redis``) and @@ -35,25 +36,21 @@ # The reason of a StreamError is the proto enum, as in the StreamErrorDetails of a failed call. from temporalio.api.enums.v1 import StreamErrorReason +from temporalio.streams._conflict import StreamIDConflictPolicy from temporalio.streams._errors import ( StreamAlreadyExistsError, StreamClosedError, StreamConfigError, StreamError, - StreamExpiredError, StreamFencedError, StreamNotFoundError, - StreamNotOwnerError, - StreamOffsetTooHighError, - StreamSeqDivergentError, - StreamStorageUnavailableError, - StreamTooLargeError, - StreamUnsupportedError, ) from temporalio.streams._items import END, End, ItemKind, Record, Superseded +from temporalio.streams._offset import StreamOffset from temporalio.streams._plugin import StreamBackend -from temporalio.streams._reader import Subscription -from temporalio.streams._ref import StreamOptions, StreamRef +from temporalio.streams._reader import Reader +from temporalio.streams._ref import StreamRef +from temporalio.streams._stream_client import producer # nexgen generates these wire models. Refs, Records and Superseded items carry them. from temporalio.streams.system.models import ( @@ -93,6 +90,7 @@ "ProducerEpochKindLocalActivity", "ProducerEpochKindStandaloneActivity", "ProducerEpochKindWorkflow", + "Reader", "Record", "StreamAlreadyExistsError", "StreamBackend", @@ -100,21 +98,15 @@ "StreamConfigError", "StreamError", "StreamErrorReason", - "StreamExpiredError", "StreamFencedError", + "StreamIDConflictPolicy", "StreamNotFoundError", - "StreamNotOwnerError", - "StreamOffsetTooHighError", - "StreamOptions", + "StreamOffset", "StreamRef", - "StreamSeqDivergentError", - "StreamStorageUnavailableError", "StreamSuperseded", - "StreamTooLargeError", - "StreamUnsupportedError", "StreamWriter", - "Subscription", "Superseded", "Timestamp", "WorkflowTaskEpoch", + "producer", ] diff --git a/temporalio/streams/_access.py b/temporalio/streams/_access.py index 15951fb360..2aec4baca2 100644 --- a/temporalio/streams/_access.py +++ b/temporalio/streams/_access.py @@ -12,7 +12,8 @@ from temporalio.streams import _convert, _reader from temporalio.streams._backend import BackendCalls, CallOptions from temporalio.streams._items import End -from temporalio.streams._reader import Item, Position, Subscription +from temporalio.streams._offset import StreamOffset +from temporalio.streams._reader import Item, Position, Reader from temporalio.streams._writer import RecordWriter from temporalio.streams.system import models @@ -27,13 +28,13 @@ class Access: stream_id: str writer: Callable[[], RecordWriter] - def publish(self, values: Sequence[Any]) -> asyncio.Future[int]: + def append(self, values: Sequence[Any]) -> asyncio.Future[StreamOffset]: """Start the write of the values as Records, in order, and return its future. The future gives the offset of the last Record. """ if not values: - raise TypeError("publish takes one or more values") + raise TypeError("append takes one or more values") return self.writer().write(values) async def items( @@ -121,15 +122,15 @@ def read_spec( ) -def subscription( +def reader( items: Callable[[Position], AsyncGenerator[Item, None]], read: ReadSpec, options: CallOptions | None = None, -) -> Subscription[Any]: +) -> Reader[Any]: """A read over the items that ``items`` gives.""" # A read from END or of the last Records has no resume offset until its first page. if read.last is not None or read.offset is End.END: position = Position(None) else: position = Position(read.offset or 0) - return Subscription(items(position), position, options) + return Reader(items(position), position, options) diff --git a/temporalio/streams/_activity.py b/temporalio/streams/_activity.py index f20c8660be..1b900804be 100644 --- a/temporalio/streams/_activity.py +++ b/temporalio/streams/_activity.py @@ -14,7 +14,7 @@ if TYPE_CHECKING: import temporalio.client - from temporalio.client import BufferedPublisher + from temporalio.client import BufferedAppender @dataclass @@ -32,8 +32,8 @@ class ActivityStreams: previous_completed_task: stream_v1.WorkflowTaskEpoch | None = None failed_tasks: tuple[int, ...] = () writers: dict[str, RecordWriter] = field(default_factory=dict) - # The attempt's BufferedPublishers, which the end of the attempt flushes. - publishers: list[BufferedPublisher[Any]] = field(default_factory=list) + # The attempt's BufferedAppenders, which the end of the attempt flushes. + appenders: list[BufferedAppender[Any]] = field(default_factory=list) # The Worker's client. Its Streams in the attempt write as the attempt, and other clients' # Streams write as those clients. client: temporalio.client.Client | None = None @@ -54,14 +54,14 @@ def writer(self, stream_id: str) -> RecordWriter: return writer async def close(self) -> None: - """Flush the attempt's publishers, then close every writer of the attempt. + """Flush the attempt's appenders, then close every writer of the attempt. A write error that no caller got fails the attempt. """ errors: list[BaseException] = [] - for publisher in self.publishers: + for appender in self.appenders: try: - await publisher._flush_at_attempt_end() + await appender._flush_at_attempt_end() except Exception as err: errors.append(err) for writer in self.writers.values(): diff --git a/temporalio/streams/_activity_stream.py b/temporalio/streams/_activity_stream.py index d820eea229..863983a529 100644 --- a/temporalio/streams/_activity_stream.py +++ b/temporalio/streams/_activity_stream.py @@ -10,9 +10,10 @@ from temporalio.client import Stream from temporalio.streams import _activity, _backend, _errors -from temporalio.streams._access import Access, ReadSpec, subscription +from temporalio.streams._access import Access, ReadSpec, reader from temporalio.streams._items import Record, Superseded -from temporalio.streams._reader import Subscription +from temporalio.streams._offset import StreamOffset +from temporalio.streams._reader import Reader from temporalio.streams._ref import StreamRef from temporalio.streams._stream_client import StreamCalls @@ -52,10 +53,10 @@ def future_done(future: concurrent.futures.Future[_R]) -> None: future.add_done_callback(future_done) -def _not_owner(ref: StreamRef[Any]) -> _errors.StreamNotOwnerError: +def _not_owner(ref: StreamRef[Any]) -> _errors.StreamError: # A Workflow or a client closes a Stream, and the Store refuses an Activity's close. Code in # an Activity only writes, so the refusal comes before any call. - return _errors.StreamNotOwnerError( + return _errors.not_owner( f"Can't close the Stream {ref.id} from an Activity. Only Workflow code or a client " "closes a Stream." ) @@ -65,7 +66,7 @@ def _check_loop() -> None: try: asyncio.get_running_loop() except RuntimeError: - raise _errors.StreamUnsupportedError( + raise _errors.unsupported( "A sync Activity can't wait for a Stream call. Wrap the Stream in a SyncStream." ) from None @@ -95,27 +96,27 @@ def _access(self) -> Access: writer=lambda: streams.writer(stream_id), ) - def publish( + def append( self, values: tuple[Any, ...], rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None, - ) -> asyncio.Future[int]: + ) -> asyncio.Future[StreamOffset]: """Start a write. The write's task takes the call options.""" _check_loop() with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): - return self._access().publish(values) + return self._access().append(values) - def subscribe( + def read( self, read: ReadSpec, rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None, - ) -> Subscription[Any]: + ) -> Reader[Any]: """Start a read.""" _check_loop() access = self._access() - return subscription( + return reader( lambda position: access.items(read, position), read, _backend.options_of(rpc_metadata, rpc_timeout), @@ -123,15 +124,15 @@ def subscribe( async def get_offset( self, rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None - ) -> int: + ) -> StreamOffset: """Get the next offset.""" with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): - return await self._access().get_offset() + return StreamOffset._from_wire(await self._access().get_offset()) async def close( self, rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None ) -> None: - """Raise :py:class:`StreamNotOwnerError`, since an Activity can't close a Stream.""" + """Raise a Stream error, since an Activity can't close a Stream.""" raise _not_owner(self._ref) async def enter(self) -> None: @@ -142,12 +143,6 @@ async def exit(self, exc: BaseException | None) -> None: """Raise, since an Activity can't close a Stream.""" raise _not_owner(self._ref) - def producer(self, name: str, epoch: int) -> StreamCalls: - """Raise, since an Activity writes as its attempt.""" - raise _errors.StreamUnsupportedError( - "A Stream in an Activity writes as the attempt, not as a Producer with a name" - ) - def attempt_of(stream: Stream[Any]) -> _activity.ActivityStreams | None: """The Activity attempt that ``stream`` writes as, or None for a client's Stream.""" @@ -202,7 +197,7 @@ def run_blocking(call: Coroutine[Any, Any, _R]) -> _R: def sync_items( - items: Subscription[T], + items: Reader[T], ) -> Generator[Record[T] | Superseded, None, None]: """The items of a read, each of which blocks the Activity's thread.""" try: @@ -216,5 +211,5 @@ def sync_items( run_blocking(items.aclose()) -async def _next(items: Subscription[T]) -> Record[T] | Superseded: +async def _next(items: Reader[T]) -> Record[T] | Superseded: return await items.__anext__() diff --git a/temporalio/streams/_conflict.py b/temporalio/streams/_conflict.py new file mode 100644 index 0000000000..27f3c63617 --- /dev/null +++ b/temporalio/streams/_conflict.py @@ -0,0 +1,26 @@ +"""The conflict policy of a Stream create.""" + +from __future__ import annotations + +from enum import IntEnum + +import temporalio.api.enums.v1 + + +class StreamIDConflictPolicy(IntEnum): + """How a create of a Stream whose id already exists is handled. + + ``FAIL`` raises :py:class:`temporalio.streams.StreamAlreadyExistsError`. ``USE_EXISTING`` + returns the existing Stream and ignores the new options. See + :py:class:`temporalio.api.enums.v1.StreamIdConflictPolicy`. + + .. warning:: + This API is experimental. + """ + + FAIL = int( + temporalio.api.enums.v1.StreamIdConflictPolicy.STREAM_ID_CONFLICT_POLICY_FAIL + ) + USE_EXISTING = int( + temporalio.api.enums.v1.StreamIdConflictPolicy.STREAM_ID_CONFLICT_POLICY_USE_EXISTING + ) diff --git a/temporalio/streams/_errors.py b/temporalio/streams/_errors.py index 67bf05d352..4024341ee5 100644 --- a/temporalio/streams/_errors.py +++ b/temporalio/streams/_errors.py @@ -15,17 +15,15 @@ "StreamClosedError", "StreamConfigError", "StreamError", - "StreamExpiredError", "StreamFencedError", "StreamNotFoundError", - "StreamNotOwnerError", - "StreamOffsetTooHighError", - "StreamSeqDivergentError", - "StreamStorageUnavailableError", - "StreamTooLargeError", - "StreamUnsupportedError", ] +# The one reason a retry can fix. Every other reason raises a non-retryable error. +_RETRYABLE_REASONS: frozenset[int] = frozenset( + {StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE} +) + class StreamConfigError(RuntimeError): """The Stream backend config is wrong, such as when no backend has a given name. @@ -52,8 +50,9 @@ def no_default_backend( class StreamError(temporalio.exceptions.ApplicationError): """A failed Stream call, with its ``StreamErrorReason``. - A reason with no class of its own, such as ``STREAM_ERROR_REASON_STORAGE_CORRUPT``, raises - this class. + A reason with no class of its own raises this class. The reason tells the cases apart, such + as ``STREAM_ERROR_REASON_NOT_OWNER`` or ``STREAM_ERROR_REASON_STORAGE_UNAVAILABLE``. A Store + outage is the one retryable error. .. warning:: This API is experimental. @@ -62,7 +61,6 @@ class StreamError(temporalio.exceptions.ApplicationError): _reason: ClassVar[StreamErrorReason.ValueType] = ( StreamErrorReason.STREAM_ERROR_REASON_UNSPECIFIED ) - _retryable: ClassVar[bool] = False def __init__( self, message: str, *, reason: StreamErrorReason.ValueType | None = None @@ -70,12 +68,12 @@ def __init__( """Create the error. ``reason`` defaults to the reason of the class.""" reason = self._reason if reason is None else reason # The details carry the reason, so the error keeps it when it crosses an Activity or a - # Nexus operation. + # Nexus operation. The reason, not the class, decides whether a retry can fix it. super().__init__( message, temporalio.api.stream.v1.StreamErrorDetails(reason=reason), type=type(self).__name__, - non_retryable=not self._retryable, + non_retryable=reason not in _RETRYABLE_REASONS, ) self.reason: StreamErrorReason.ValueType = reason @@ -90,16 +88,6 @@ class StreamNotFoundError(StreamError): _reason = StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND -class StreamExpiredError(StreamError): - """The Stream's Retention or safety TTL passed. - - .. warning:: - This API is experimental. - """ - - _reason = StreamErrorReason.STREAM_ERROR_REASON_EXPIRED - - class StreamClosedError(StreamError): """The Stream is closed. @@ -120,92 +108,34 @@ class StreamFencedError(StreamError): _reason = StreamErrorReason.STREAM_ERROR_REASON_FENCED -class StreamSeqDivergentError(StreamError): - """A batch has the sequence of the last batch of its epoch, and other content. - - .. warning:: - This API is experimental. - """ - - _reason = StreamErrorReason.STREAM_ERROR_REASON_SEQ_DIVERGENT - - -class StreamNotOwnerError(StreamError): - """A create gives another owner than the Stream has, or the owner refuses this close. - - .. warning:: - This API is experimental. - """ - - _reason = StreamErrorReason.STREAM_ERROR_REASON_NOT_OWNER - - -class StreamOffsetTooHighError(StreamError): - """The offset is past the newest offset + 1. - - A read from it works again once the Stream has Records up to it. - - .. warning:: - This API is experimental. - """ - - _reason = StreamErrorReason.STREAM_ERROR_REASON_OFFSET_TOO_HIGH - - -class StreamTooLargeError(StreamError): - """A Record or a batch is over the Store's limit. - - .. warning:: - This API is experimental. - """ - - _reason = StreamErrorReason.STREAM_ERROR_REASON_TOO_LARGE - - -class StreamStorageUnavailableError(StreamError): - """The Store failed or did not answer. It is the only retryable Stream error. - - .. warning:: - This API is experimental. - """ - - _reason = StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE - _retryable = True - - -class StreamUnsupportedError(StreamError): - """The Store or the code that makes the call lacks a capability that the call needs. +class StreamAlreadyExistsError(StreamError): + """A create of an id that exists, with no earlier create of the same request. .. warning:: This API is experimental. """ - _reason = StreamErrorReason.STREAM_ERROR_REASON_UNSUPPORTED + _reason = StreamErrorReason.STREAM_ERROR_REASON_ALREADY_EXISTS -class StreamAlreadyExistsError(StreamError): - """A create of an id that exists, with no earlier create of the same request. +def not_owner(message: str) -> StreamError: + """A ``StreamError`` for code that may not close a Stream it does not own.""" + return StreamError(message, reason=StreamErrorReason.STREAM_ERROR_REASON_NOT_OWNER) - .. warning:: - This API is experimental. - """ - _reason = StreamErrorReason.STREAM_ERROR_REASON_ALREADY_EXISTS +def unsupported(message: str) -> StreamError: + """A ``StreamError`` for a call that the code or the Store does not support.""" + return StreamError( + message, reason=StreamErrorReason.STREAM_ERROR_REASON_UNSUPPORTED + ) _ERRORS: dict[int, type[StreamError]] = { error._reason: error for error in ( StreamNotFoundError, - StreamExpiredError, StreamClosedError, StreamFencedError, - StreamSeqDivergentError, - StreamNotOwnerError, - StreamOffsetTooHighError, - StreamTooLargeError, - StreamStorageUnavailableError, - StreamUnsupportedError, StreamAlreadyExistsError, ) } diff --git a/temporalio/streams/_items.py b/temporalio/streams/_items.py index eddbc45535..881aaaa052 100644 --- a/temporalio/streams/_items.py +++ b/temporalio/streams/_items.py @@ -6,6 +6,7 @@ from dataclasses import dataclass, field from typing import Any, Generic, Literal, TypeVar +from temporalio.streams._offset import StreamOffset from temporalio.streams.system import models T = TypeVar("T") @@ -48,7 +49,7 @@ class Record(Generic[T]): data: T """Value of the Record, decoded with the Stream's payload type. A list is one value.""" - offset: int + offset: StreamOffset """Position of the Record in the Stream. Offsets are ordered and unique, with gaps.""" producer: models.Producer | None @@ -74,13 +75,13 @@ class Superseded: kind: Literal[ItemKind.SUPERSEDED] = field(default=ItemKind.SUPERSEDED, init=False) """Always :py:attr:`ItemKind.SUPERSEDED`.""" - offset: int + offset: StreamOffset """Position of the item in the Stream.""" facts: models.StreamSuperseded """The withdrawal facts from the Store. Its ``writer`` gives the Records that it withdraws.""" - withdrawn_offsets: tuple[int, ...] = () + withdrawn_offsets: tuple[StreamOffset, ...] = () """The offsets of the Records that this read gave and this item withdraws, in order. A read gives each offset once. It doesn't include Records from before the read's start. diff --git a/temporalio/streams/_offset.py b/temporalio/streams/_offset.py new file mode 100644 index 0000000000..0e6471d921 --- /dev/null +++ b/temporalio/streams/_offset.py @@ -0,0 +1,91 @@ +"""The opaque offset of a Record in a Stream.""" + +from __future__ import annotations + +from typing import Any + +from typing_extensions import Self + + +class StreamOffset: + """An opaque, ordered position of a Record in one Stream. + + A read gives it on each :py:class:`Record`, an append returns the offset of its last Record, + and :py:meth:`get_offset` returns the next one. Code compares two offsets of the same Stream + with ``<``, ``<=``, ``>``, ``>=`` and ``==``, and resumes a read from one with + ``read(offset=...)``. It has no meaning across Streams, and no arithmetic. To store or send + one, use :py:meth:`__str__` and :py:meth:`parse`. + + .. warning:: + This API is experimental. + """ + + __slots__ = ("_offset",) + + def __init__(self, offset: int) -> None: + """Wrap a wire offset. Build one with :py:meth:`parse`, or take one from a call.""" + self._offset = offset + + @classmethod + def _from_wire(cls, offset: int) -> Self: + """The offset of a wire int64, for the read and append plumbing.""" + return cls(offset) + + @property + def _wire(self) -> int: + """The wire int64, for the read and append plumbing.""" + return self._offset + + @classmethod + def parse(cls, text: str) -> StreamOffset: + """The offset of a string that :py:meth:`__str__` wrote. + + Raises: + ValueError: The string is not a Stream offset. + """ + try: + return cls(int(text)) + except ValueError: + raise ValueError(f"Not a Stream offset: {text!r}") from None + + def __str__(self) -> str: + """The string form, which :py:meth:`parse` reads back.""" + return str(self._offset) + + def __repr__(self) -> str: + """Show the offset.""" + return f"StreamOffset({self._offset})" + + def __eq__(self, other: object) -> bool: + """Two offsets of one Stream are equal when they are the same position.""" + if not isinstance(other, StreamOffset): + return NotImplemented + return self._offset == other._offset + + def __lt__(self, other: Any) -> bool: + """Order two offsets of one Stream.""" + if not isinstance(other, StreamOffset): + return NotImplemented + return self._offset < other._offset + + def __le__(self, other: Any) -> bool: + """Order two offsets of one Stream.""" + if not isinstance(other, StreamOffset): + return NotImplemented + return self._offset <= other._offset + + def __gt__(self, other: Any) -> bool: + """Order two offsets of one Stream.""" + if not isinstance(other, StreamOffset): + return NotImplemented + return self._offset > other._offset + + def __ge__(self, other: Any) -> bool: + """Order two offsets of one Stream.""" + if not isinstance(other, StreamOffset): + return NotImplemented + return self._offset >= other._offset + + def __hash__(self) -> int: + """Hash of the position.""" + return hash(self._offset) diff --git a/temporalio/streams/_reader.py b/temporalio/streams/_reader.py index b101476f50..9c4c3a2606 100644 --- a/temporalio/streams/_reader.py +++ b/temporalio/streams/_reader.py @@ -20,6 +20,7 @@ producer_key, withdraws, ) +from temporalio.streams._offset import StreamOffset from temporalio.streams.system import models T = TypeVar("T") @@ -42,7 +43,7 @@ class Position: next_offset: int | None -class Subscription(Generic[T]): +class Reader(Generic[T]): """The items of one read of a Stream, in order. Iterate it with ``async for``. It yields :py:class:`Record` items, and :py:class:`Superseded` items that withdraw earlier @@ -59,7 +60,7 @@ def __init__( position: Position, options: _backend.CallOptions | None = None, ) -> None: - """Create a read. Use a Stream's ``subscribe`` instead.""" + """Create a read. Use a Stream's ``read`` instead.""" self._items = items self._position = position # A client read's rpc_metadata and rpc_timeout, for each call of each step. @@ -73,7 +74,7 @@ def __init__( ] = {} @property - def next_offset(self) -> int | None: + def next_offset(self) -> StreamOffset | None: """The offset to resume from, after the last item or the last page that the read got. Pass it as ``offset`` to a later read. After a ``Superseded`` item, it's that item's own @@ -82,9 +83,10 @@ def next_offset(self) -> int | None: Record that failed to decode, it's past that Record. It's None before the first page of a read from ``END`` or of a read with ``last``. """ - return self._position.next_offset + wire = self._position.next_offset + return StreamOffset._from_wire(wire) if wire is not None else None - def __aiter__(self) -> Subscription[T]: + def __aiter__(self) -> Reader[T]: """The read is its own iterator.""" return self @@ -97,12 +99,13 @@ async def __anext__(self) -> Record[T] | Superseded: """ with _backend.using(self._options): item = await self._items.__anext__() + offset = item.offset._wire if isinstance(item, Superseded): # The Record of the same entry has the same offset and comes next. - self._position.next_offset = item.offset + self._position.next_offset = offset else: self._position.next_offset = max( - self._position.next_offset or 0, item.offset + 1 + self._position.next_offset or 0, offset + 1 ) if ( isinstance(item, Record) @@ -113,17 +116,17 @@ async def __anext__(self) -> Record[T] | Superseded: key = producer_key(item.producer) _, runs = self._shown.setdefault(key, (item.producer, array.array("Q"))) # A batch has consecutive offsets, so a run of them costs one first and last pair. - if runs and runs[-1] + 1 == item.offset: - runs[-1] = item.offset + if runs and runs[-1] + 1 == offset: + runs[-1] = offset else: - runs.extend((item.offset, item.offset)) + runs.extend((offset, offset)) elif isinstance(item, Superseded): item = dataclasses.replace( item, withdrawn_offsets=self._withdraw(item.facts) ) return item - def _withdraw(self, facts: models.StreamSuperseded) -> tuple[int, ...]: + def _withdraw(self, facts: models.StreamSuperseded) -> tuple[StreamOffset, ...]: offsets: list[int] = [] for key, (producer, runs) in list(self._shown.items()): if withdraws(facts, producer): @@ -131,7 +134,7 @@ def _withdraw(self, facts: models.StreamSuperseded) -> tuple[int, ...]: offsets.extend(range(first, last + 1)) # A withdrawn Record stays withdrawn, so the read forgets it. del self._shown[key] - return tuple(sorted(offsets)) + return tuple(StreamOffset._from_wire(offset) for offset in sorted(offsets)) async def aclose(self) -> None: """End the read early, from the task that reads it. It makes no more calls. @@ -308,6 +311,10 @@ async def _item( return None case models.StreamItemData(value=data): value = _convert.decode(converter, data.body, payload_type) - return Record(data=value, offset=offset, producer=data.producer) + return Record( + data=value, + offset=StreamOffset._from_wire(offset), + producer=data.producer, + ) case models.StreamItemSuperseded(value=facts): - return Superseded(offset=offset, facts=facts) + return Superseded(offset=StreamOffset._from_wire(offset), facts=facts) diff --git a/temporalio/streams/_ref.py b/temporalio/streams/_ref.py index ebbdf14cc4..d87414bcec 100644 --- a/temporalio/streams/_ref.py +++ b/temporalio/streams/_ref.py @@ -3,11 +3,10 @@ from __future__ import annotations import typing -from datetime import timedelta from typing import Any, Generic, TypeVar import google.protobuf.json_format -from typing_extensions import Self, TypedDict +from typing_extensions import Self import temporalio.api.stream.v1 as stream_v1 import temporalio.converter @@ -16,21 +15,6 @@ T = TypeVar("T") -class StreamOptions(TypedDict, total=False): - """The options of a new Stream. ``create_or_get_stream`` ignores them on a Stream that exists. - - .. warning:: - This API is experimental. - """ - - retention: timedelta - """How long a Record stays readable, even while the Stream is open. - - A closed Stream is kept for one Retention after its close. Defaults to the backend's - Retention. - """ - - def _model(ref: StreamRef[Any]) -> models.StreamRef: return models.StreamRef(id=ref.id, owner=ref.owner, backend=ref.backend) @@ -89,7 +73,7 @@ class StreamRef(Generic[T]): dataclass or a pydantic model, in the proto JSON form of that message. A ref has no calls. To make calls, Workflow code gets a Stream with - ``workflow.to_stream(ref)``, and other code with ``client.to_stream(ref)``. + ``workflow.get_stream(ref)``, and other code with ``client.get_stream(ref)``. .. warning:: This API is experimental. diff --git a/temporalio/streams/_stream_client.py b/temporalio/streams/_stream_client.py index f986523c07..4a3d920bf8 100644 --- a/temporalio/streams/_stream_client.py +++ b/temporalio/streams/_stream_client.py @@ -4,11 +4,14 @@ import abc import asyncio +import contextlib +import contextvars import itertools import os +import typing import uuid import weakref -from collections.abc import AsyncGenerator, Mapping +from collections.abc import AsyncGenerator, Iterator, Mapping from datetime import timedelta from typing import Any, TypeVar @@ -16,11 +19,13 @@ import temporalio.client from temporalio.client._streams import Stream from temporalio.streams import _activity, _backend, _errors, _ids -from temporalio.streams._access import Access, ReadSpec, subscription +from temporalio.streams._access import Access, ReadSpec, reader from temporalio.streams._backend import BackendCalls +from temporalio.streams._conflict import StreamIDConflictPolicy +from temporalio.streams._offset import StreamOffset from temporalio.streams._plugin import client_backend -from temporalio.streams._reader import Item, Position, Subscription -from temporalio.streams._ref import StreamOptions, StreamRef +from temporalio.streams._reader import Item, Position, Reader +from temporalio.streams._ref import StreamRef from temporalio.streams._writer import RecordWriter from temporalio.streams.system import models @@ -34,6 +39,11 @@ weakref.WeakKeyDictionary() ) +# The named Producer that client writes use, set by the producer() context manager. +_producer_override: contextvars.ContextVar[tuple[str, int] | None] = ( + contextvars.ContextVar("__temporal_stream_producer", default=None) +) + def _new_process_key() -> None: # A forked child has the parent's key and count, so two children would give their clients @@ -50,6 +60,35 @@ def _new_process_key() -> None: _register_at_fork(after_in_child=_new_process_key) +@contextlib.contextmanager +def producer(name: str, epoch: int) -> Iterator[None]: + """Make the client writes in the block write as the Producer ``name`` at ``epoch``. + + Use it for a writer that must fence its earlier instances, such as a server that restarts. + Keep ``epoch`` in durable storage, and raise it at each start. After a write at a higher + epoch, a write at a lower one raises :py:class:`temporalio.streams.StreamFencedError`. The + Records of the earlier epochs aren't withdrawn, since they're real messages. Run only one + live writer for each name. Writes in an Activity ignore it and write as the attempt. + + Args: + name: The Producer name, the same across restarts. + epoch: The writer's epoch, 1 or more. + + Raises: + ValueError: The epoch is below 1. + + .. warning:: + This API is experimental. + """ + if epoch < 1: + raise ValueError("A Producer's epoch is 1 or more") + token = _producer_override.set((name, epoch)) + try: + yield + finally: + _producer_override.reset(token) + + def _client_epoch(attempt: int) -> stream_v1.ProducerEpoch: return stream_v1.ProducerEpoch(client=stream_v1.ClientEpoch(attempt=attempt)) @@ -75,27 +114,27 @@ class StreamCalls(abc.ABC): """The calls of one :py:class:`Stream`, such as in client code.""" @abc.abstractmethod - def publish( + def append( self, values: tuple[Any, ...], rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None, - ) -> asyncio.Future[int]: - """Start a write. See :py:meth:`Stream.publish`.""" + ) -> asyncio.Future[StreamOffset]: + """Start a write. See :py:meth:`Stream.append`.""" @abc.abstractmethod - def subscribe( + def read( self, read: ReadSpec, rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None, - ) -> Subscription[Any]: - """Start a read. See :py:meth:`Stream.subscribe`.""" + ) -> Reader[Any]: + """Start a read. See :py:meth:`Stream.read`.""" @abc.abstractmethod async def get_offset( self, rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None - ) -> int: + ) -> StreamOffset: """Get the next offset. See :py:meth:`Stream.get_offset`.""" @abc.abstractmethod @@ -112,16 +151,12 @@ async def enter(self) -> None: async def exit(self, exc: BaseException | None) -> None: """End an ``async with`` block of the Stream. See :py:meth:`Stream.__aexit__`.""" - @abc.abstractmethod - def producer(self, name: str, epoch: int) -> StreamCalls: - """The calls of a Producer with a name of its own. See :py:meth:`Stream.producer`.""" - class ClientCalls(StreamCalls): """The calls of one Stream in client code. - Each publish starts its write at the call. All Streams of one client and Stream id write as - one Producer, unless a Stream is of a Producer with a name of its own. + Each append starts its write at the call. All Streams of one client and Stream id write as + one Producer, unless :py:func:`producer` sets a Producer with a name of its own. """ def __init__( @@ -129,14 +164,13 @@ def __init__( client: temporalio.client.Client, backend: BackendCalls, ref: StreamRef[Any], - named: tuple[str, int] | None = None, ) -> None: - """Create the calls of ``ref``. ``named`` is a Producer's name and epoch.""" + """Create the calls of ``ref``.""" self._client = client self._backend = backend self._ref = ref - self._named = named - self._writer: RecordWriter | None = None + # One writer for each Producer this Stream wrote as, keyed by the producer() override. + self._writers: dict[tuple[str, int] | None, RecordWriter] = {} self._closed = False def _access(self) -> Access: @@ -145,14 +179,18 @@ def _access(self) -> Access: stream_id = self._ref.id def writer() -> RecordWriter: - if self._writer is None: - self._writer = RecordWriter( + # Read the Producer at write time, so a write in a producer() block uses it. + named = _producer_override.get() + found = self._writers.get(named) + if found is None: + found = RecordWriter( writers=backend.writers(client.namespace, client.data_converter), stream_id=stream_id, - producer=_client_producer(client, self._named), + producer=_client_producer(client, named), free=backend.free_writer, ) - return self._writer + self._writers[named] = found + return found return Access( backend=backend, @@ -165,30 +203,34 @@ def writer() -> RecordWriter: def _raise_lost_write(self) -> None: # Client code has no end, such as a run's, to raise a write error that no caller got. So # the Stream's next call raises it. - error = self._writer.unretrieved_error() if self._writer else None - if error is not None: - raise error - - def publish( + first: BaseException | None = None + for writer in list(self._writers.values()): + error = writer.unretrieved_error() + if error is not None and first is None: + first = error + if first is not None: + raise first + + def append( self, values: tuple[Any, ...], rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None, - ) -> asyncio.Future[int]: + ) -> asyncio.Future[StreamOffset]: """Start a write. The write's task takes the call options.""" self._raise_lost_write() with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): - return self._access().publish(values) + return self._access().append(values) - def subscribe( + def read( self, read: ReadSpec, rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None, - ) -> Subscription[Any]: + ) -> Reader[Any]: """Start a read.""" self._raise_lost_write() - return subscription( + return reader( lambda position: self._items(read, position), read, _backend.options_of(rpc_metadata, rpc_timeout), @@ -206,17 +248,17 @@ async def _items( async def get_offset( self, rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None - ) -> int: + ) -> StreamOffset: """Get the next offset.""" self._raise_lost_write() with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): - return await self._access().get_offset() + return StreamOffset._from_wire(await self._access().get_offset()) def _check_close(self) -> None: if self._ref.owner is not None: # Only the run chain closes a Stream bound to a Workflow. The Store refuses a # client's close too, but this check makes no call. - raise _errors.StreamNotOwnerError( + raise _errors.not_owner( f"Can't close the Stream {self._ref.id} from a client, since it is bound to a " "Workflow" ) @@ -244,12 +286,6 @@ async def exit(self, exc: BaseException | None) -> None: if not self._closed: await self.close({}, None) - def producer(self, name: str, epoch: int) -> StreamCalls: - """The calls of the same Stream and client as a Producer with a name of its own.""" - if epoch < 1: - raise ValueError("A Producer's epoch is 1 or more") - return ClientCalls(self._client, self._backend, self._ref, (name, epoch)) - async def create( client: temporalio.client.Client, @@ -257,29 +293,27 @@ async def create( *, id: str, backend: str | None, - options: StreamOptions, - or_get: bool, + id_conflict_policy: StreamIDConflictPolicy, + rpc_metadata: Mapping[str, str | bytes], + rpc_timeout: timedelta | None, ) -> Stream[T]: - """Create a Stream, or with ``or_get`` get the one that exists. + """Create a Stream. - See :py:attr:`temporalio.client.Client.create_stream`. + See :py:meth:`temporalio.client.Client.create_stream`. """ if not id: # An empty id would fail only at the Store, with an unclear error. raise ValueError("The Stream id is empty") backend_name, calls = client_backend(client, backend) - policy = ( - models.StreamIdConflictPolicy.StreamIdConflictPolicyUseExisting - if or_get - else models.StreamIdConflictPolicy.StreamIdConflictPolicyFail - ) - # The generated create sends a new request id for each call. A closed, expired or another - # owner's Stream fails either create. - await calls.service(client.namespace).create_stream[object]( + # Delegate to the generated client. It sends a new request id for each call. A closed, + # expired or another owner's Stream fails the create. + await calls.service(client.namespace).create_stream( id=id, backend=backend_name, - retention=options.get("retention"), - id_conflict_policy=policy, + id_conflict_policy=models.StreamIdConflictPolicy(int(id_conflict_policy)), + body_type=object, + rpc_metadata=rpc_metadata, + rpc_timeout=rpc_timeout, ) ref: StreamRef[T] = StreamRef(id=id, backend=backend_name) return _stream(client, calls, ref, payload_type) @@ -290,31 +324,33 @@ def get_stream( payload_type: type[T] | None, *, id: str, + owner: models.Execution | None, backend: str | None, ) -> Stream[T]: """Get a Stream by its id. - See :py:attr:`temporalio.client.Client.get_stream`. + See :py:meth:`temporalio.client.Client.get_stream`. """ if not id: # An empty id would fail only at the first call, with an unclear error. raise ValueError("The Stream id is empty") backend_name, calls = client_backend(client, backend) - ref: StreamRef[T] = StreamRef(id=id, backend=backend_name) + ref: StreamRef[T] = StreamRef(id=id, owner=owner, backend=backend_name) return _stream(client, calls, ref, payload_type) -def to_stream( +def from_ref( client: temporalio.client.Client, ref: StreamRef[T], - *, - payload_type: type[T] | None, ) -> Stream[T]: """The Stream of a ref that came as data. - See :py:meth:`temporalio.client.Client.to_stream`. + See :py:meth:`temporalio.client.Client.get_stream`. """ _, calls = client_backend(client, ref.backend, hint="the ref's backend field") + # A ref that code built as StreamRef[T] carries the payload type, so reads decode with it. + arguments = typing.get_args(getattr(ref, "__orig_class__", None)) + payload_type = arguments[0] if arguments else None return _stream(client, calls, ref, payload_type) diff --git a/temporalio/streams/_workflow.py b/temporalio/streams/_workflow.py index 1c8d871c78..c45293e72b 100644 --- a/temporalio/streams/_workflow.py +++ b/temporalio/streams/_workflow.py @@ -28,7 +28,9 @@ import temporalio.nexus.system._support as _support import temporalio.workflow from temporalio.streams import _errors -from temporalio.streams._ref import StreamOptions, StreamRef +from temporalio.streams._conflict import StreamIDConflictPolicy +from temporalio.streams._offset import StreamOffset +from temporalio.streams._ref import StreamRef from temporalio.streams._writer import WriteFuture, unretrieved_error from temporalio.streams.system import models from temporalio.streams.system.workflow._resources import ( @@ -68,7 +70,7 @@ class _Write: # Record count up to its own last Record, and its future. activity: str request: google.protobuf.message.Message - calls: list[tuple[int, asyncio.Future[int]]] + calls: list[tuple[int, WriteFuture]] size: int = 0 started: bool = False @@ -106,9 +108,9 @@ class _RunState: uses_streams: bool = False # The async with blocks whose close waits for the run's outcome. deferred_closes: list[StreamRef[Any]] = field(default_factory=list) - # The publishes that failed before a caller got the error. A caller can still get it, so + # The appends that failed before a caller got the error. A caller can still get it, so # the run's end checks them again. - failed_publishes: list[asyncio.Future[int]] = field(default_factory=list) + failed_publishes: list[asyncio.Future[StreamOffset]] = field(default_factory=list) # The calls of the generated Workflow API in flight. A call ends after its write, so the # run's end waits for both. calls: set[asyncio.Future[Any]] = field(default_factory=set) @@ -214,7 +216,7 @@ def _records(request: google.protobuf.message.Message) -> int: def _write( stream_id: str, activity: str, request: google.protobuf.message.Message -) -> asyncio.Future[int]: +) -> asyncio.Future[StreamOffset]: state = _state() loop = asyncio.get_running_loop() future = WriteFuture(loop=loop) @@ -310,7 +312,7 @@ async def run() -> None: _errors.from_reason(int(answer["reason"]), str(answer["message"])) ) else: - call.set_result(int(answer)) + call._set_wire(int(answer)) task = loop.create_task(run()) task.add_done_callback(lambda _: _raise_nowait_error(state)) @@ -319,10 +321,10 @@ async def run() -> None: return future -def _write_nowait(future: asyncio.Future[int]) -> None: +def _write_nowait(future: asyncio.Future[StreamOffset]) -> None: state = _state() - def done(future: asyncio.Future[int]) -> None: + def done(future: asyncio.Future[StreamOffset]) -> None: if future.cancelled() or future.exception() is None: return if state.nowait_error is None: @@ -424,10 +426,12 @@ def _check( return run_stream -def _start_append(ref: StreamRef[Any], values: tuple[Any, ...]) -> asyncio.Future[int]: +def _start_append( + ref: StreamRef[Any], values: tuple[Any, ...] +) -> asyncio.Future[StreamOffset]: # The checks raise at the call, as the write's place in the order does. if not values: - raise TypeError("publish takes one or more values") + raise TypeError("append takes one or more values") _check(ref, "write to") call = _support.start_now( generated_stream.append_records(ref=_generated_ref(ref), records=list(values)) @@ -435,9 +439,11 @@ def _start_append(ref: StreamRef[Any], values: tuple[Any, ...]) -> asyncio.Futur return _call_future(call, int) -def _append(ref: StreamRef[Any], values: tuple[Any, ...]) -> asyncio.Future[int]: +def _append( + ref: StreamRef[Any], values: tuple[Any, ...] +) -> asyncio.Future[StreamOffset]: if not values: - raise TypeError("publish takes one or more values") + raise TypeError("append takes one or more values") _check(ref, "write to") records = [stream_v1.AppendRecord(body=_payload(v)) for v in values] return _write( @@ -447,7 +453,7 @@ def _append(ref: StreamRef[Any], values: tuple[Any, ...]) -> asyncio.Future[int] ) -def _start_close(ref: StreamRef[Any]) -> asyncio.Future[int]: +def _start_close(ref: StreamRef[Any]) -> asyncio.Future[StreamOffset]: _closing(ref) call = _support.start_now(generated_stream.close_stream(ref=_generated_ref(ref))) return _call_future(call, lambda _: 0) @@ -468,7 +474,7 @@ def check_owner(ref: StreamRef[Any]) -> None: ): # The backend refuses the close too, but this check fails before the Workflow # schedules a command. - raise _errors.StreamNotOwnerError( + raise _errors.not_owner( f"Can't close the Stream {ref.id}, since it is bound to another run chain" ) @@ -478,33 +484,35 @@ def _closing(ref: StreamRef[Any]) -> _RunStream: return _check(ref, "close") -def _close(ref: StreamRef[Any]) -> asyncio.Future[int]: +def _close(ref: StreamRef[Any]) -> asyncio.Future[StreamOffset]: _closing(ref).closed = True return _write( ref.id, CLOSE_ACTIVITY, stream_pb.CloseStreamRequest(ref=_proto_ref(ref)) ) -def publish(ref: StreamRef[Any], values: tuple[Any, ...]) -> asyncio.Future[int]: +def append( + ref: StreamRef[Any], values: tuple[Any, ...] +) -> asyncio.Future[StreamOffset]: """Start the write Local Activity. An error that no caller gets fails the run. - See :py:meth:`temporalio.workflow.WorkflowStream.publish`. + See :py:meth:`temporalio.workflow.WorkflowStream.append`. """ state = _state() future = _start_append(ref, values) loop = asyncio.get_running_loop() - def warn(future: asyncio.Future[int]) -> None: + def warn(future: asyncio.Future[StreamOffset]) -> None: error = unretrieved_error(future) if error is not None: temporalio.workflow.logger.warning( - "A publish to the Stream %s failed, and no caller has its error. If no " + "An append to the Stream %s failed, and no caller has its error. If no " "caller gets the error, it fails the run when the run ends: %s", ref.id, error, ) - def failed(future: asyncio.Future[int]) -> None: + def failed(future: asyncio.Future[StreamOffset]) -> None: if unretrieved_error(future) is not None: state.failed_publishes.append(future) # A caller that already waits for the future gets the error before warn runs. @@ -538,7 +546,7 @@ async def exit_block(ref: StreamRef[Any], exc: BaseException | None) -> None: state.deferred_closes.append(ref) -def publish_nowait(ref: StreamRef[Any], values: tuple[Any, ...]) -> None: +def append_nowait(ref: StreamRef[Any], values: tuple[Any, ...]) -> None: """Start the write. Its error raises in the run as an error that the run doesn't catch.""" _write_nowait(_start_append(ref, values)) @@ -613,12 +621,10 @@ def _cancelled(exc: BaseException) -> bool: ) -def _conflict_policy(or_get: bool) -> models.StreamIdConflictPolicy: - return ( - models.StreamIdConflictPolicy.StreamIdConflictPolicyUseExisting - if or_get - else models.StreamIdConflictPolicy.StreamIdConflictPolicyFail - ) +def _models_policy( + policy: StreamIDConflictPolicy, +) -> models.StreamIdConflictPolicy: + return models.StreamIdConflictPolicy(int(policy)) async def create_stream( @@ -626,19 +632,18 @@ async def create_stream( id: str, *, backend: str | None, - options: StreamOptions, - or_get: bool, + id_conflict_policy: StreamIDConflictPolicy, ) -> WorkflowStream[T]: - """Create a standalone Stream, or with ``or_get`` get the one that exists. + """Create a standalone Stream. See :py:data:`temporalio.workflow.create_stream`. """ ref: StreamRef[T] = _creating(id, backend) - await generated_create.create_stream[payload_type or object]( + await generated_create.create_stream( id=id, backend=ref.backend, - retention=options.get("retention"), - id_conflict_policy=_conflict_policy(or_get), + id_conflict_policy=_models_policy(id_conflict_policy), + body_type=payload_type or object, ) return WorkflowStream(ref) @@ -648,20 +653,19 @@ def create_stream_nowait( id: str, *, backend: str | None, - options: StreamOptions, - or_get: bool, + id_conflict_policy: StreamIDConflictPolicy, ) -> WorkflowStream[T]: - """Create a standalone Stream, or get it, without an await. + """Create a standalone Stream without an await. See :py:data:`temporalio.workflow.create_stream_nowait`. """ ref: StreamRef[T] = _creating(id, backend) call = _support.start_now( - generated_create.create_stream[payload_type or object]( + generated_create.create_stream( id=id, backend=ref.backend, - retention=options.get("retention"), - id_conflict_policy=_conflict_policy(or_get), + id_conflict_policy=_models_policy(id_conflict_policy), + body_type=payload_type or object, ) ) _write_nowait(_call_future(call, lambda _: 0)) @@ -680,7 +684,7 @@ def _creating(id: str, backend: str | None) -> StreamRef[Any]: def _call_future( call: asyncio.Task[Any], result: Callable[[Any], int] -) -> asyncio.Future[int]: +) -> asyncio.Future[StreamOffset]: # start_now runs a call of the generated Workflow API up to its first wait, so its write # takes its place in the order at once. The call gives the write's result, or the handle # that gives it. The caller's future follows both, and the run's end waits for them. @@ -688,7 +692,7 @@ def _call_future( future = WriteFuture(loop=loop) async def follow() -> None: - # The task sets the future before it ends, so the run's end sees a failed publish. + # The task sets the future before it ends, so the run's end sees a failed append. try: started = await call if isinstance(started, temporalio.workflow.NexusOperationHandle): @@ -699,7 +703,7 @@ async def follow() -> None: except BaseException as err: future.set_exception(_errors.from_failure(err) or err) else: - future.set_result(result(started)) + future._set_wire(result(started)) task = loop.create_task(follow()) state = _state() @@ -711,7 +715,7 @@ async def follow() -> None: def _start_create( ref: StreamRef[Any], request: models.CreateStreamRequest -) -> asyncio.Future[int]: +) -> asyncio.Future[StreamOffset]: _check(ref, "create", backend_checked=True) proto = models._CreateStreamRequestTransferTypeConverter().to_transfer_type(request) proto.stream_ref.CopyFrom(_proto_ref(ref)) @@ -729,7 +733,7 @@ class _WriteHandle(temporalio.workflow.NexusOperationHandle[OutputT]): # The interceptor runs a call of the generated Workflow API as a write Local Activity, so # the call's handle waits for the write. def __init__( - self, write: asyncio.Future[int], result: Callable[[int], OutputT] + self, write: asyncio.Future[StreamOffset], result: Callable[[int], OutputT] ) -> None: self._write = write self._result = result @@ -742,7 +746,7 @@ def __await__(self) -> Generator[Any, Any, OutputT]: return self._wait().__await__() async def _wait(self) -> OutputT: - return self._result(await self._write) + return self._result((await self._write)._wire) @property def operation_token(self) -> str | None: @@ -791,14 +795,14 @@ def run_close_stream( return _WriteHandle(write, lambda _: models.CloseStreamResponse()) -def no_workflow_reads() -> _errors.StreamUnsupportedError: +def no_workflow_reads() -> _errors.StreamError: """The error of a read or a describe in Workflow code.""" - return _errors.StreamUnsupportedError( + return _errors.unsupported( "Workflow code can't read a Stream. Read it from an Activity." ) -def to_stream(ref: StreamRef[T]) -> WorkflowStream[T]: +def get_stream(ref: StreamRef[T]) -> WorkflowStream[T]: """Get the Stream of a ref that came as data. See :py:func:`temporalio.workflow.to_stream`. diff --git a/temporalio/streams/_writer.py b/temporalio/streams/_writer.py index 94c06f98ac..b7f68c680b 100644 --- a/temporalio/streams/_writer.py +++ b/temporalio/streams/_writer.py @@ -8,14 +8,16 @@ from typing import TYPE_CHECKING, Any import temporalio.api.stream.v1 as stream_v1 +from temporalio.api.enums.v1 import StreamErrorReason from temporalio.streams import _errors +from temporalio.streams._offset import StreamOffset if TYPE_CHECKING: from temporalio.streams.writer_system import client as writer_client from temporalio.streams.writer_system.client import Writer -class WriteFuture(asyncio.Future[int]): +class WriteFuture(asyncio.Future[StreamOffset]): """The future of a write. A caller that gets its result or its error sets ``retrieved``. A write error that no caller gets still raises, at the end of a Workflow run or an Activity @@ -24,8 +26,12 @@ class WriteFuture(asyncio.Future[int]): retrieved = False - def result(self) -> int: - """The last offset of the write.""" + def _set_wire(self, offset: int) -> None: + """Set the result from a wire offset. The plumbing works in wire ints.""" + self.set_result(StreamOffset._from_wire(offset)) + + def result(self) -> StreamOffset: + """The offset of the write's last Record.""" self.retrieved = True return super().result() @@ -34,7 +40,7 @@ def exception(self) -> BaseException | None: self.retrieved = True return super().exception() - def __await__(self) -> Generator[Any, None, int]: + def __await__(self) -> Generator[Any, None, StreamOffset]: """Wait for the write, and return its last offset.""" # The await of asyncio's future doesn't call result(), so this one does. if not self.done(): @@ -45,7 +51,7 @@ def __await__(self) -> Generator[Any, None, int]: __iter__ = __await__ -def unretrieved_error(future: asyncio.Future[int]) -> BaseException | None: +def unretrieved_error(future: asyncio.Future[StreamOffset]) -> BaseException | None: """The error of a failed write that no caller got, else None.""" if not isinstance(future, WriteFuture) or future.retrieved or future.cancelled(): return None @@ -99,12 +105,13 @@ async def withdraw_earlier_attempts(self) -> None: """ try: await self.open() - except ( - _errors.StreamNotFoundError, - _errors.StreamClosedError, - _errors.StreamExpiredError, - ): - pass + except _errors.StreamError as err: + if err.reason not in ( + StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND, + StreamErrorReason.STREAM_ERROR_REASON_CLOSED, + StreamErrorReason.STREAM_ERROR_REASON_EXPIRED, + ): + raise async def open(self) -> Writer: """Open the crate's writer once. @@ -172,7 +179,7 @@ def done(task: asyncio.Task[int]) -> None: future.set_exception(error) self._failed.append(future) else: - future.set_result(task.result()) + future._set_wire(task.result()) task.add_done_callback(done) return future diff --git a/temporalio/streams/_writes.py b/temporalio/streams/_writes.py index 873f614fa1..153e0d88be 100644 --- a/temporalio/streams/_writes.py +++ b/temporalio/streams/_writes.py @@ -15,6 +15,7 @@ import temporalio.api.stream.v1 as stream_v1 import temporalio.api.streamservice.v1 as stream_pb import temporalio.common +from temporalio.api.enums.v1 import StreamErrorReason from temporalio.streams import _activity, _backend, _errors, _ids from temporalio.streams._workflow import ( APPEND_ACTIVITY, @@ -234,7 +235,7 @@ async def append_records( ) answer = response.last_offset or 0 except _errors.StreamError as err: - if not isinstance(err, _errors.StreamTooLargeError): + if err.reason != StreamErrorReason.STREAM_ERROR_REASON_TOO_LARGE: if err.non_retryable: _PENDING.pop(key, epoch) raise @@ -258,7 +259,7 @@ async def close_stream(request: stream_pb.CloseStreamRequest) -> int: if owner is not None and owner.business_id != streams.workflow_id: # The Store refuses an Activity Producer's close, so the Activity closes as the run. It # refuses a Stream bound to another Workflow, and Workflow code checks the run chain. - raise _errors.StreamNotOwnerError( + raise _errors.not_owner( f"Can't close the Stream {request.ref.id} for another Workflow" ) await service.send_close_stream( diff --git a/temporalio/streams/system/client/_extensions.py b/temporalio/streams/system/client/_extensions.py index 1ff83f68c3..62d8001ef8 100644 --- a/temporalio/streams/system/client/_extensions.py +++ b/temporalio/streams/system/client/_extensions.py @@ -24,37 +24,68 @@ _NexusServiceT = typing.TypeVar("_NexusServiceT") BodyT = typing.TypeVar("BodyT") -_BodyT = typing.TypeVar("_BodyT") -class _GetStream(typing.Generic[BodyT]): - """Subscript it with the type arguments that the call decodes with.""" +class StreamServiceClientExtension(abc.ABC): + """Starts the operations of ``StreamService`` from code outside a Workflow. - def __init__( - self, - extension: StreamServiceClientExtension, - body_type: type[BodyT] | None = None, - ) -> None: - self._extension: StreamServiceClientExtension = extension - self.body_type: type[BodyT] = typing.cast( - "type[BodyT]", typing.Any if body_type is None else body_type - ) + A client class inherits this class and provides the SDK's ``create_nexus_client`` + method. - def __getitem__(self, body_type: type[_BodyT]) -> _GetStream[_BodyT]: - return _GetStream(self._extension, body_type) + .. warning:: + This API is experimental and subject to change. + """ + + @abc.abstractmethod + def create_nexus_client( + self, + service: type[_NexusServiceT] | str, + endpoint: str, + ) -> temporalio.client.NexusClient[_NexusServiceT]: + """Return the Nexus client that starts the operations of ``service``.""" + ... - def __call__( + @typing.overload + def get_stream( self, + ref: StreamRef[BodyT], + /, *, - id: str, - owner: Execution | None = None, - backend: str | None = None, created: bool | None = None, + ) -> Stream[BodyT]: ... + @typing.overload + def get_stream( + self, + ref: StreamRef, + *, + created: bool | None = None, + body_type: type[BodyT], + ) -> Stream[BodyT]: ... + @typing.overload + def get_stream( + self, + *, + ref: StreamRef, + created: bool | None = None, + body_type: None = None, + ) -> Stream[object]: ... + def get_stream( + self, + ref: StreamRef[BodyT], + *, + created: bool | None = None, + body_type: type[BodyT] | None = None, ) -> Stream[BodyT]: - """A handle on an existing ``Stream``, with no call.""" - ref = StreamRef(id=id, owner=owner, backend=backend) - return Stream[self.body_type]( - self._extension.create_nexus_client( + """Return the ``Stream`` handle for this ref.""" + arguments = typing.get_args(getattr(ref, "__orig_class__", None)) + # None means no explicit type, so the ref's type argument decides, else Any. + if body_type is None: + body_type = typing.cast( + "type[BodyT]", + arguments[0] if arguments else typing.Any, + ) + return Stream[body_type]( + self.create_nexus_client( service=StreamService, endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, ), @@ -62,25 +93,67 @@ def __call__( created=created, ) - -class _CreateStream(typing.Generic[BodyT]): - """Subscript it with the type arguments that the call decodes with.""" - - def __init__( + async def _stream_service_create_stream( self, - extension: StreamServiceClientExtension, - body_type: type[BodyT] | None = None, - ) -> None: - self._extension: StreamServiceClientExtension = extension - self.body_type: type[BodyT] = typing.cast( - "type[BodyT]", typing.Any if body_type is None else body_type + request: CreateStreamRequest, + body_type: type[BodyT], + *, + rpc_metadata: collections.abc.Mapping[str, str | bytes] | None = None, + rpc_timeout: datetime.timedelta | None = None, + ) -> Stream[BodyT]: + nexus_client = self.create_nexus_client( + service=StreamService, + endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, + ) + handle = await nexus_client.start_operation( + operation=StreamService.create_stream, + arg=request, + id=f"{_uuid.uuid4()}", + rpc_metadata=rpc_metadata or {}, + rpc_timeout=rpc_timeout, ) + result = await handle.result() + if result.ref is None: + raise RuntimeError("CreateStream answered without `ref`") + resource = Stream[body_type]( + nexus_client, + ref=result.ref, + created=result.created or None, + ) + return resource + + @typing.overload + @resource_call + async def create_stream( + self, + *, + id: str, + owner: Execution | None = None, + backend: str | None = None, + retention: datetime.timedelta | None = None, + id_conflict_policy: StreamIdConflictPolicy | None = None, + body_type: type[BodyT], + rpc_metadata: collections.abc.Mapping[str, str | bytes] | None = None, + rpc_timeout: datetime.timedelta | None = None, + ) -> Stream[BodyT]: ... - def __getitem__(self, body_type: type[_BodyT]) -> _CreateStream[_BodyT]: - return _CreateStream(self._extension, body_type) + @typing.overload + @resource_call + async def create_stream( + self, + *, + id: str, + owner: Execution | None = None, + backend: str | None = None, + retention: datetime.timedelta | None = None, + id_conflict_policy: StreamIdConflictPolicy | None = None, + body_type: None = None, + rpc_metadata: collections.abc.Mapping[str, str | bytes] | None = None, + rpc_timeout: datetime.timedelta | None = None, + ) -> Stream[object]: ... @resource_call - async def __call__( + async def create_stream( self, *, id: str, @@ -88,9 +161,10 @@ async def __call__( backend: str | None = None, retention: datetime.timedelta | None = None, id_conflict_policy: StreamIdConflictPolicy | None = None, + body_type: typing.Any = None, rpc_metadata: collections.abc.Mapping[str, str | bytes] | None = None, rpc_timeout: datetime.timedelta | None = None, - ) -> Stream[BodyT]: + ) -> Stream[typing.Any]: """ Args: id: The Stream id, 1 to 4096 bytes. @@ -103,6 +177,8 @@ async def __call__( id_conflict_policy: What the create does when the Stream exists. Unspecified is fail. """ + if body_type is None: + body_type = typing.Any stream_ref = StreamRef( id=id, owner=owner, @@ -115,65 +191,6 @@ async def __call__( id_conflict_policy=id_conflict_policy, request_id=_support.request_id(), ) - return await self._extension._stream_service_create_stream( - request, self.body_type, rpc_metadata=rpc_metadata, rpc_timeout=rpc_timeout - ) - - -class StreamServiceClientExtension(abc.ABC): - """Starts the operations of ``StreamService`` from code outside a Workflow. - - A client class inherits this class and provides the SDK's ``create_nexus_client`` - method. - - .. warning:: - This API is experimental and subject to change. - """ - - @abc.abstractmethod - def create_nexus_client( - self, - service: type[_NexusServiceT] | str, - endpoint: str, - ) -> temporalio.client.NexusClient[_NexusServiceT]: - """Return the Nexus client that starts the operations of ``service``.""" - ... - - @property - def get_stream(self) -> _GetStream[typing.Any]: - """Subscript it with the type arguments that the call decodes with.""" - return _GetStream(self) - - async def _stream_service_create_stream( - self, - request: CreateStreamRequest, - body_type: type[BodyT], - *, - rpc_metadata: collections.abc.Mapping[str, str | bytes] | None = None, - rpc_timeout: datetime.timedelta | None = None, - ) -> Stream[BodyT]: - nexus_client = self.create_nexus_client( - service=StreamService, - endpoint=temporalio.nexus.system.TEMPORAL_SYSTEM_ENDPOINT, - ) - handle = await nexus_client.start_operation( - operation=StreamService.create_stream, - arg=request, - id=f"{_uuid.uuid4()}", - rpc_metadata=rpc_metadata or {}, - rpc_timeout=rpc_timeout, + return await self._stream_service_create_stream( + request, body_type, rpc_metadata=rpc_metadata, rpc_timeout=rpc_timeout ) - result = await handle.result() - if result.ref is None: - raise RuntimeError("CreateStream answered without `ref`") - resource = Stream[body_type]( - nexus_client, - ref=result.ref, - created=result.created or None, - ) - return resource - - @property - def create_stream(self) -> _CreateStream[typing.Any]: - """Subscript it with the type arguments that the call decodes with.""" - return _CreateStream(self) diff --git a/temporalio/streams/system/client/_resources/stream.py b/temporalio/streams/system/client/_resources/stream.py index 5b6fb222c3..fd5d4c8101 100644 --- a/temporalio/streams/system/client/_resources/stream.py +++ b/temporalio/streams/system/client/_resources/stream.py @@ -34,7 +34,6 @@ class Stream(typing.Generic[BodyT]): """One Stream. Build it from a stream-ref, or get it from a create.""" - ref: StreamRef created: bool | None def __init__( @@ -42,11 +41,11 @@ def __init__( nexus_client: temporalio.client.NexusClient[typing.Any], /, *, - ref: StreamRef, + ref: StreamRef[BodyT], created: bool | None, ) -> None: self._nexus_client: temporalio.client.NexusClient[typing.Any] = nexus_client - self.ref = ref + self._ref: StreamRef[BodyT] = ref self.created = created @property @@ -63,6 +62,17 @@ def body_type(self) -> type[BodyT]: ) return typing.cast("type[BodyT]", arguments[0] if arguments else typing.Any) + @property + def ref(self) -> StreamRef[BodyT]: + # The ref carries the handle's payload type, so a reader of it keeps the type. + if not typing.get_args(getattr(self._ref, "__orig_class__", None)): + self._ref = StreamRef[self.body_type]( + id=self._ref.id, + owner=self._ref.owner, + backend=self._ref.backend, + ) + return self._ref + async def __aenter__(self) -> typing_extensions.Self: return self diff --git a/temporalio/streams/system/models.py b/temporalio/streams/system/models.py index 7c1e9a7a19..b89b436144 100644 --- a/temporalio/streams/system/models.py +++ b/temporalio/streams/system/models.py @@ -62,6 +62,7 @@ def _stream_status_from_proto(value: int) -> StreamStatus: BodyT = typing.TypeVar("BodyT") +_BodyT = typing_extensions.TypeVar("_BodyT", default=typing.Any) @dataclasses.dataclass(slots=True) @@ -140,8 +141,8 @@ def to_transfer_type( )(CreateStreamRequest) # pyright: ignore[reportUnusedCallResult] -@dataclasses.dataclass(slots=True) -class StreamRef: +@dataclasses.dataclass +class StreamRef(typing.Generic[_BodyT]): """Identifies one Stream. The id is opaque to callers.""" id: str @@ -167,7 +168,7 @@ def from_transfer_type( if not value.id: raise ValueError("missing required field StreamRef.id") id = value.id - return StreamRef( + return type_hint( id=id, owner=_ExecutionTransferTypeConverter().from_transfer_type( value.owner, Execution diff --git a/temporalio/streams/system/workflow/_resources/stream.py b/temporalio/streams/system/workflow/_resources/stream.py index 632d148b5c..7e24fd6bdd 100644 --- a/temporalio/streams/system/workflow/_resources/stream.py +++ b/temporalio/streams/system/workflow/_resources/stream.py @@ -28,21 +28,19 @@ ) BodyT = typing.TypeVar("BodyT") -_BodyT = typing.TypeVar("_BodyT") class Stream(typing.Generic[BodyT]): """One Stream. Build it from a stream-ref, or get it from a create.""" - ref: StreamRef created: bool | None def __init__( self, - ref: StreamRef, + ref: StreamRef[BodyT], created: bool | None, ) -> None: - self.ref = ref + self._ref: StreamRef[BodyT] = ref self.created = created @property @@ -59,6 +57,17 @@ def body_type(self) -> type[BodyT]: ) return typing.cast("type[BodyT]", arguments[0] if arguments else typing.Any) + @property + def ref(self) -> StreamRef[BodyT]: + # The ref carries the handle's payload type, so a reader of it keeps the type. + if not typing.get_args(getattr(self._ref, "__orig_class__", None)): + self._ref = StreamRef[self.body_type]( + id=self._ref.id, + owner=self._ref.owner, + backend=self._ref.backend, + ) + return self._ref + async def __aenter__(self) -> typing_extensions.Self: return self @@ -126,37 +135,65 @@ async def describe( return await handle -class _GetStream(typing.Generic[BodyT]): - """Subscript it with the type arguments that the call decodes with.""" +@typing.overload +def get_stream( + *, + ref: StreamRef, + created: bool | None = None, + body_type: type[BodyT], +) -> Stream[BodyT]: ... - def __init__( - self, - body_type: type[BodyT] | None = None, - ) -> None: - self.body_type: type[BodyT] = typing.cast( - "type[BodyT]", typing.Any if body_type is None else body_type - ) - def __getitem__(self, body_type: type[_BodyT]) -> _GetStream[_BodyT]: - return _GetStream(body_type) +@typing.overload +def get_stream( + *, + ref: StreamRef, + created: bool | None = None, + body_type: None = None, +) -> Stream[object]: ... - def __call__( - self, - *, - id: str, - owner: Execution | None = None, - backend: str | None = None, - created: bool | None = None, - ) -> Stream[BodyT]: - """A handle on an existing ``Stream``, with no call.""" - ref = StreamRef(id=id, owner=owner, backend=backend) - return Stream[self.body_type]( - ref=ref, - created=created, - ) +@typing.overload +def get_stream( + *, + id: str, + owner: Execution | None = None, + backend: str | None = None, + created: bool | None = None, + body_type: type[BodyT], +) -> Stream[BodyT]: ... -get_stream: _GetStream[typing.Any] = _GetStream() + +@typing.overload +def get_stream( + *, + id: str, + owner: Execution | None = None, + backend: str | None = None, + created: bool | None = None, + body_type: None = None, +) -> Stream[object]: ... + + +def get_stream( + *, + ref: StreamRef | None = None, + id: str | None = None, + owner: Execution | None = None, + backend: str | None = None, + created: bool | None = None, + body_type: typing.Any = None, +) -> Stream[typing.Any]: + """A handle on an existing ``Stream``, with no call.""" + if ref is None: + assert id is not None + ref = StreamRef(id=id, owner=owner, backend=backend) + if body_type is None: + body_type = typing.Any + return Stream[body_type]( + ref=ref, + created=created, + ) async def _append_records( @@ -236,61 +273,72 @@ async def _read_records( ) -class _ReadRecords(typing.Generic[BodyT]): - """Subscript it with the type arguments that the call decodes with.""" +@typing.overload +async def read_records( + *, + ref: StreamRef, + from_offset: int | None = None, + start: StreamReadStart | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + create_wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + body_type: type[BodyT], +) -> temporalio.workflow.NexusOperationHandle[ReadRecordsResponse[BodyT],]: ... - def __init__( - self, - body_type: type[BodyT] | None = None, - ) -> None: - self.body_type: type[BodyT] = typing.cast( - "type[BodyT]", typing.Any if body_type is None else body_type - ) - def __getitem__(self, body_type: type[_BodyT]) -> _ReadRecords[_BodyT]: - return _ReadRecords(body_type) +@typing.overload +async def read_records( + *, + ref: StreamRef, + from_offset: int | None = None, + start: StreamReadStart | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + create_wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + body_type: None = None, +) -> temporalio.workflow.NexusOperationHandle[ReadRecordsResponse[object],]: ... - async def __call__( - self, - *, - ref: StreamRef, - from_offset: int | None = None, - start: StreamReadStart | None = None, - max_records: int | None = None, - wait: datetime.timedelta | None = None, - create_wait: datetime.timedelta | None = None, - max_bytes: int | None = None, - ) -> temporalio.workflow.NexusOperationHandle[ReadRecordsResponse[BodyT],]: - """A long poll. NOT_FOUND for an uncreated Stream. Workflow code reads from a Local - Activity. - - Args: - from_offset: Start offset. Empty uses start. An offset past the newest one + - 1 fails. - start: Where to start when from-offset is empty. Unspecified is the - beginning. - max_records: About this many items per page, 1 to 1000. A Superseded item - stays with its Record. - wait: How long to wait for a new Record before an empty answer. At most 60 - s. - create_wait: How long to wait for an uncreated Stream. Empty fails at once. - At most 60 s. - max_bytes: The maximum Record bytes in a page. A larger Record is still - returned alone. - """ - request = ReadRecordsRequest( - ref=ref, - from_offset=from_offset, - start=start, - max_records=max_records, - wait=wait, - create_wait=create_wait, - max_bytes=max_bytes, - ) - return await _read_records(request, self.body_type) +async def read_records( + *, + ref: StreamRef, + from_offset: int | None = None, + start: StreamReadStart | None = None, + max_records: int | None = None, + wait: datetime.timedelta | None = None, + create_wait: datetime.timedelta | None = None, + max_bytes: int | None = None, + body_type: typing.Any = None, +) -> temporalio.workflow.NexusOperationHandle[ReadRecordsResponse[typing.Any],]: + """A long poll. NOT_FOUND for an uncreated Stream. Workflow code reads from a Local + Activity. -read_records: _ReadRecords[typing.Any] = _ReadRecords() + Args: + from_offset: Start offset. Empty uses start. An offset past the newest one + 1 + fails. + start: Where to start when from-offset is empty. Unspecified is the beginning. + max_records: About this many items per page, 1 to 1000. A Superseded item stays + with its Record. + wait: How long to wait for a new Record before an empty answer. At most 60 s. + create_wait: How long to wait for an uncreated Stream. Empty fails at once. At + most 60 s. + max_bytes: The maximum Record bytes in a page. A larger Record is still returned + alone. + """ + if body_type is None: + body_type = typing.Any + request = ReadRecordsRequest( + ref=ref, + from_offset=from_offset, + start=start, + max_records=max_records, + wait=wait, + create_wait=create_wait, + max_bytes=max_bytes, + ) + return await _read_records(request, body_type) async def _describe_stream( diff --git a/temporalio/streams/system/workflow/operations/create_stream.py b/temporalio/streams/system/workflow/operations/create_stream.py index 291f8a0e35..a1b5e8c6d2 100644 --- a/temporalio/streams/system/workflow/operations/create_stream.py +++ b/temporalio/streams/system/workflow/operations/create_stream.py @@ -20,7 +20,6 @@ from .._resources import Stream BodyT = typing.TypeVar("BodyT") -_BodyT = typing.TypeVar("_BodyT") async def _create_stream( @@ -45,55 +44,65 @@ async def _create_stream( ) -class _CreateStream(typing.Generic[BodyT]): - """Subscript it with the type arguments that the call decodes with.""" - - def __init__( - self, - body_type: type[BodyT] | None = None, - ) -> None: - self.body_type: type[BodyT] = typing.cast( - "type[BodyT]", typing.Any if body_type is None else body_type - ) +@typing.overload +@resource_call +async def create_stream( + *, + id: str, + owner: Execution | None = None, + backend: str | None = None, + retention: datetime.timedelta | None = None, + id_conflict_policy: StreamIdConflictPolicy | None = None, + body_type: type[BodyT], +) -> Stream[BodyT]: ... - def __getitem__(self, body_type: type[_BodyT]) -> _CreateStream[_BodyT]: - return _CreateStream(body_type) - @resource_call - async def __call__( - self, - *, - id: str, - owner: Execution | None = None, - backend: str | None = None, - retention: datetime.timedelta | None = None, - id_conflict_policy: StreamIdConflictPolicy | None = None, - ) -> Stream[BodyT]: - """ - Args: - id: The Stream id, 1 to 4096 bytes. - owner: The run chain or Standalone Activity of a bound Stream. Empty if - standalone. - backend: Registered backend name, empty for the default. The Store never - reads it. - retention: The age after which a Record is dropped. Empty is the Store - default. - id_conflict_policy: What the create does when the Stream exists. Unspecified - is fail. - """ - stream_ref = StreamRef( - id=id, - owner=owner, - backend=backend, - ) - request = CreateStreamRequest( - stream_ref=stream_ref, - retention=retention, - withdraw_run=_support.withdraw_run(), - id_conflict_policy=id_conflict_policy, - request_id=_support.request_id(), - ) - return await _create_stream(request, self.body_type) +@typing.overload +@resource_call +async def create_stream( + *, + id: str, + owner: Execution | None = None, + backend: str | None = None, + retention: datetime.timedelta | None = None, + id_conflict_policy: StreamIdConflictPolicy | None = None, + body_type: None = None, +) -> Stream[object]: ... -create_stream: _CreateStream[typing.Any] = _CreateStream() +@resource_call +async def create_stream( + *, + id: str, + owner: Execution | None = None, + backend: str | None = None, + retention: datetime.timedelta | None = None, + id_conflict_policy: StreamIdConflictPolicy | None = None, + body_type: typing.Any = None, +) -> Stream[typing.Any]: + """ + Args: + id: The Stream id, 1 to 4096 bytes. + owner: The run chain or Standalone Activity of a bound Stream. Empty if + standalone. + backend: Registered backend name, empty for the default. The Store never reads + it. + retention: The age after which a Record is dropped. Empty is the Store default. + id_conflict_policy: What the create does when the Stream exists. Unspecified is + fail. + """ + if body_type is None: + body_type = typing.Any + stream_ref = StreamRef( + id=id, + owner=owner, + backend=backend, + ) + request = CreateStreamRequest( + stream_ref=stream_ref, + retention=retention, + withdraw_run=_support.withdraw_run(), + id_conflict_policy=id_conflict_policy, + request_id=_support.request_id(), + ) + return await _create_stream(request, body_type) diff --git a/temporalio/workflow/__init__.py b/temporalio/workflow/__init__.py index d55d65996a..691f925666 100644 --- a/temporalio/workflow/__init__.py +++ b/temporalio/workflow/__init__.py @@ -153,11 +153,9 @@ ) from ._streams import ( WorkflowStream, - create_or_get_stream, - create_or_get_stream_nowait, create_stream, create_stream_nowait, - to_stream, + get_stream, ) from ._workflow_ops import ( ChildWorkflowCancellationType, @@ -267,11 +265,9 @@ "logger", "unsafe", "WorkflowStream", - "create_or_get_stream", - "create_or_get_stream_nowait", "create_stream", "create_stream_nowait", - "to_stream", + "get_stream", "ChildWorkflowCancellationType", "ChildWorkflowConfig", "ChildWorkflowHandle", diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 136f759c6a..03d126bce9 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -8,27 +8,25 @@ import asyncio from types import TracebackType -from typing import TYPE_CHECKING, Any, Generic, TypeVar +from typing import TYPE_CHECKING, Any, Generic, TypeVar, overload -from typing_extensions import Self, Unpack +from typing_extensions import Self if TYPE_CHECKING: import temporalio.api.stream.v1 - from temporalio.streams import StreamOptions, StreamRef + from temporalio.streams import StreamIDConflictPolicy, StreamOffset, StreamRef T = TypeVar("T") -_U = TypeVar("_U") class WorkflowStream(Generic[T]): """A Stream in Workflow code, with its calls. ``T`` is the payload type. - Get one with :py:data:`create_stream`, :py:data:`create_or_get_stream`, or - :py:func:`to_stream` for a ref that came as data. Its :py:attr:`ref` is the data to pass - on, such as to an Activity, a child Workflow or the next run of continue-as-new, or through - a query. + Get one with :py:func:`create_stream` or :py:func:`get_stream` for a ref that came as data. + Its :py:attr:`ref` is the data to pass on, such as to an Activity, a child Workflow or the + next run of continue-as-new, or through a query. - Workflow code writes to and closes a Stream, and can't read it. Each publish is a Local + Workflow code writes to and closes a Stream, and can't read it. Each append is a Local Activity, and each close a command that the backend runs after the Workflow Task commits. .. warning:: @@ -36,7 +34,7 @@ class WorkflowStream(Generic[T]): """ def __init__(self, ref: StreamRef[T]) -> None: - """Create a Stream. Use ``create_stream`` or ``to_stream`` instead.""" + """Create a Stream. Use ``create_stream`` or ``get_stream`` instead.""" self._ref = ref @property @@ -51,18 +49,18 @@ def proto_ref(self) -> temporalio.api.stream.v1.StreamRef: return _ref.to_proto(self._ref) - def publish(self, *values: T) -> asyncio.Future[int]: + def append(self, *values: T) -> asyncio.Future[StreamOffset]: """Start the write of one Record for each value, in order, and return its future. The write has its place in the Stream's order when the call returns, also when no caller awaits the future. The writes to one Stream keep the order of the calls. A cancel of the future doesn't stop the write. A list is one value. - The write is a Local Activity, so History has its result and not the values. Publishes + The write is a Local Activity, so History has its result and not the values. Appends that wait behind a write to the same Stream are sent as one Local Activity, and each call gets the offset of its own last Record. A caller that awaits the future, or calls its ``result()`` or ``exception()``, gets the write's error and owns it. An error that - no caller gets raises at the end of the run, as an error of :py:meth:`publish_nowait` + no caller gets raises at the end of the run, as an error of :py:meth:`append_nowait` does. So a ``StreamError`` fails the run. The Workflow's logger logs the error when the write fails, if no caller waits for it then. @@ -83,9 +81,9 @@ def publish(self, *values: T) -> asyncio.Future[int]: """ from temporalio.streams import _workflow - return _workflow.publish(self._ref, values) + return _workflow.append(self._ref, values) - def publish_nowait(self, *values: T) -> None: + def append_nowait(self, *values: T) -> None: """Write one Record for each value, in order, without an await. Use it in sync Workflow code, such as ``@workflow.init`` and sync signal and update @@ -95,12 +93,12 @@ def publish_nowait(self, *values: T) -> None: """ from temporalio.streams import _workflow - _workflow.publish_nowait(self._ref, values) + _workflow.append_nowait(self._ref, values) async def close(self) -> None: """Close the Stream after the writes before it, once the Workflow Task commits. - A close carries no value, so code with a last value publishes it first. Readers get the + A close carries no value, so code with a last value appends it first. Readers get the Records before the close, then their read ends. Later writes in the run raise :py:class:`temporalio.streams.StreamClosedError`. @@ -111,8 +109,8 @@ async def close(self) -> None: close, and a Worker with ``no_remote_activities=True`` doesn't. Raises: - StreamNotOwnerError: The Stream is bound to another run chain, which alone closes - it. It raises with no call. + StreamError: The Stream is bound to another run chain, which alone closes it. It + raises with no call. """ from temporalio.streams import _workflow @@ -133,7 +131,7 @@ async def __aenter__(self) -> Self: """Use the Stream in ``async with``, which closes it at the end. Raises: - StreamNotOwnerError: This run chain can't close the Stream. See :py:meth:`close`. + StreamError: This run chain can't close the Stream. See :py:meth:`close`. """ from temporalio.streams import _workflow @@ -170,169 +168,126 @@ def __repr__(self) -> str: return f"{type(self).__name__}({self._ref!r})" -# Python has no type arguments on a call, so a call that takes the payload type is an object -# whose subscript binds the type, as in workflow.create_stream[Turn]("session-7-turns"). Type -# checkers then see WorkflowStream[Turn]. With no subscript, T is Any. - - -class CreateStream(Generic[T]): - """The call of :py:data:`create_stream` and :py:data:`create_or_get_stream`. - - Subscript it with the payload type, as in ``workflow.create_stream[Turn]("turns-7")``. +@overload +async def create_stream( + id: str, + *, + payload_type: type[T], + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, +) -> WorkflowStream[T]: ... + + +@overload +async def create_stream( + id: str, + *, + payload_type: None = None, + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, +) -> WorkflowStream[Any]: ... + + +async def create_stream( + id: str, + *, + payload_type: type[T] | None = None, + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, +) -> WorkflowStream[Any]: + """Create a Stream, and return it. + + Give the payload type with ``payload_type=``:: + + turns = await workflow.create_stream(f"{session_id}-turns", payload_type=Turn) + + A Stream must exist before its first append. A Workflow creates only standalone Streams, under + an id of its choice, and the Stream doesn't end with the run. Each run of a chain sends the + same create for an id, so a retry run or a cron run that creates the Stream of an earlier run + gets that Stream. A run after continue-as-new uses the Stream through the ref that the earlier + run forwards, with :py:func:`get_stream`. The create and each append are Local Activities. + + Args: + id: The Stream id, which the Workflow picks, such as a session id. + payload_type: The type that reads decode each Record with. + backend: The name of the backend. Defaults to the Worker's backend. A replay must create + the Stream on the same backend, so keep the Worker's default, or gate a new one with + ``workflow.patched``. + id_conflict_policy: ``FAIL`` raises :py:class:`StreamAlreadyExistsError` on an id that + exists, which no earlier create of this run chain made. ``USE_EXISTING`` returns the + existing Stream. + + Raises: + StreamAlreadyExistsError: ``FAIL`` on an id that exists. + ReadOnlyContextError: The code must not change state, such as a query handler. + StreamConfigError: The Worker has no Stream backend or none with that name. It fails the + Workflow Task, not the run. .. warning:: This API is experimental. """ - - def __init__(self, payload_type: type[T] | None, *, or_get: bool) -> None: - """Create the call. Use the module's ``create_stream`` instead.""" - self._payload_type = payload_type - self._or_get = or_get - - def __getitem__(self, payload_type: type[_U]) -> CreateStream[_U]: - """The call for Streams of ``payload_type``. - - It takes a class, such as ``Turn`` or ``list[str]``. Type checkers refuse a union, such - as ``str | None``, so use ``Any`` or a class that holds the value instead. - """ - return CreateStream(payload_type, or_get=self._or_get) - - async def __call__( - self, - id: str, - *, - backend: str | None = None, - **options: Unpack[StreamOptions], - ) -> WorkflowStream[T]: - """Create the Stream, and return it. - - Args: - id: The Stream id, which the Workflow picks, such as a session id. - backend: The name of the backend. Defaults to the registered backend with - ``default=True``. A replay must create the Stream on the same backend, so keep - the Worker's default, or gate a new one with ``workflow.patched``. A Worker - serves only one backend, so the name must be the Worker's backend name. - options: The options of a new Stream, such as ``retention``. See - :py:class:`temporalio.streams.StreamOptions`. - - Raises: - StreamAlreadyExistsError: ``create_stream`` of an id that exists, which no earlier - create of this run chain made. - StreamError: The backend refused the create. It's ``StreamClosedError`` or - ``StreamExpiredError`` if a Stream with that id closed or expired. - ReadOnlyContextError: The code must not change state, such as a query handler. - StreamConfigError: The Worker has no Stream backend or none with that name, or no - backend has ``default=True`` and the call gives no backend. It fails the - Workflow Task, not the run. - """ - from temporalio.streams import _workflow - - return await _workflow.create_stream( - self._payload_type, - id, - backend=backend, - options=options, - or_get=self._or_get, - ) - - -class CreateStreamNowait(Generic[T]): - """The call of :py:data:`create_stream_nowait` and :py:data:`create_or_get_stream_nowait`. - - Subscript it with the payload type, as in ``workflow.create_stream_nowait[Turn]("turns-7")``. + from temporalio.streams import StreamIDConflictPolicy, _workflow + + return await _workflow.create_stream( + payload_type, + id, + backend=backend, + id_conflict_policy=id_conflict_policy + if id_conflict_policy is not None + else StreamIDConflictPolicy.FAIL, + ) + + +@overload +def create_stream_nowait( + id: str, + *, + payload_type: type[T], + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, +) -> WorkflowStream[T]: ... + + +@overload +def create_stream_nowait( + id: str, + *, + payload_type: None = None, + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, +) -> WorkflowStream[Any]: ... + + +def create_stream_nowait( + id: str, + *, + payload_type: type[T] | None = None, + backend: str | None = None, + id_conflict_policy: StreamIDConflictPolicy | None = None, +) -> WorkflowStream[Any]: + """Start the create, and return the Stream at once. + + Use it in sync Workflow code, such as ``@workflow.init``. It's like ``asyncio.Queue.put_nowait`` + for code that can't await ``put``. It takes the arguments of :py:func:`create_stream`. A + backend error raises in the run as an error that the run doesn't catch, so a ``StreamError`` + fails the run. .. warning:: This API is experimental. """ + from temporalio.streams import StreamIDConflictPolicy, _workflow - def __init__(self, payload_type: type[T] | None, *, or_get: bool) -> None: - """Create the call. Use the module's ``create_stream_nowait`` instead.""" - self._payload_type = payload_type - self._or_get = or_get - - def __getitem__(self, payload_type: type[_U]) -> CreateStreamNowait[_U]: - """The call for Streams of ``payload_type``. It takes a class, as for ``create_stream``.""" - return CreateStreamNowait(payload_type, or_get=self._or_get) - - def __call__( - self, - id: str, - *, - backend: str | None = None, - **options: Unpack[StreamOptions], - ) -> WorkflowStream[T]: - """Start the create, and return the Stream at once. - - It takes the arguments of :py:meth:`CreateStream.__call__`. A backend error raises in - the run as an error that the run doesn't catch, so a ``StreamError`` fails the run. - """ - from temporalio.streams import _workflow - - return _workflow.create_stream_nowait( - self._payload_type, - id, - backend=backend, - options=options, - or_get=self._or_get, - ) - - -create_stream: CreateStream[Any] = CreateStream(None, or_get=False) -"""Create a Stream, which fails if a Stream of the id exists. - -Give the payload type as a type argument:: - - turns = await workflow.create_stream[Turn](f"{session_id}-turns") - -A Stream must exist before its first publish. A Workflow creates only standalone Streams, under an -id of its choice, and the Stream doesn't end with the run. Each run of a chain sends the same create -for an id, so a retry run or a cron run that creates the Stream of an earlier run gets that Stream, -and the options don't change. A run after continue-as-new uses the Stream through the ref that the -earlier run forwards, with :py:func:`to_stream`. The create and each publish are Local Activities, -and the create's error is raised here. A close runs after its Workflow Task commits, as a regular -Activity on Redis, so a task that fails first leaves the Stream open for its retry. See -:py:meth:`CreateStream.__call__`. - -.. warning:: - This API is experimental. -""" - -create_or_get_stream: CreateStream[Any] = CreateStream(None, or_get=True) -"""Create a Stream, or get the one of the id that exists. - -It takes the arguments of :py:data:`create_stream`. On a Stream that exists, it ignores the -options. It still fails on a closed or expired Stream. - -.. warning:: - This API is experimental. -""" - -create_stream_nowait: CreateStreamNowait[Any] = CreateStreamNowait(None, or_get=False) -"""Create a Stream without an await, which fails the run where :py:data:`create_stream` raises. - -Use it in sync Workflow code, such as ``@workflow.init``. It's like ``asyncio.Queue.put_nowait`` -for code that can't await ``put``. It takes the arguments of :py:data:`create_stream`. A backend -error raises in the run as an error that the run doesn't catch, so a ``StreamError`` fails the -run. - -.. warning:: - This API is experimental. -""" - -create_or_get_stream_nowait: CreateStreamNowait[Any] = CreateStreamNowait( - None, or_get=True -) -"""Create a Stream, or get the one that exists, without an await. - -It is to :py:data:`create_or_get_stream` what :py:data:`create_stream_nowait` is to -:py:data:`create_stream`. - -.. warning:: - This API is experimental. -""" + return _workflow.create_stream_nowait( + payload_type, + id, + backend=backend, + id_conflict_policy=id_conflict_policy + if id_conflict_policy is not None + else StreamIDConflictPolicy.FAIL, + ) -def to_stream(ref: StreamRef[T]) -> WorkflowStream[T]: +def get_stream(ref: StreamRef[T]) -> WorkflowStream[T]: """Get the Stream of a ref that came as data, such as an argument. It makes no call. The Stream's calls check its backend. A Worker that doesn't have it fails the Workflow Task @@ -343,4 +298,4 @@ def to_stream(ref: StreamRef[T]) -> WorkflowStream[T]: """ from temporalio.streams import _workflow - return _workflow.to_stream(ref) + return _workflow.get_stream(ref) diff --git a/tests/contrib/streams/test_activity_streams.py b/tests/contrib/streams/test_activity_streams.py index 053bf351ca..c26277cfe0 100644 --- a/tests/contrib/streams/test_activity_streams.py +++ b/tests/contrib/streams/test_activity_streams.py @@ -19,13 +19,14 @@ from temporalio import activity, streams, workflow from temporalio.api.enums.v1 import StreamErrorReason from temporalio.client import ( - BufferedPublisher, + BufferedAppender, Client, Stream, SyncStream, WorkflowHandle, ) from temporalio.common import RetryPolicy +from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.exceptions import ActivityError, ApplicationError, CancelledError from temporalio.streams import _activity, _activity_stream from temporalio.streams.system import models @@ -38,6 +39,10 @@ def own_id(name: str = "tokens") -> str: return f"{workflow.info().workflow_id}-{name}" +def reason(err: streams.StreamError) -> str: + return StreamErrorReason.Name(StreamErrorReason.ValueType(err.reason)) + + @dataclass class CallModelInput: prompt: str @@ -51,8 +56,8 @@ class SyncInput: async def read(handle: WorkflowHandle[Any, Any], name: str = "tokens") -> list[object]: - stream = handle._client.get_stream[str](f"{handle.id}-{name}") - return [item async for item in stream.subscribe(wait_new_records=False)] + stream = handle._client.get_stream(f"{handle.id}-{name}", payload_type=str) + return [item async for item in stream.read(wait_new_records=False)] async def closed(handle: WorkflowHandle[Any, Any], name: str = "tokens") -> bool: @@ -65,13 +70,13 @@ def records(items: Sequence[object]) -> list[streams.Record[Any]]: @activity.defn -async def call_model(input: CallModelInput) -> list[int]: - stream = activity.client().to_stream(input.stream) - offsets = [await stream.publish(word) for word in input.prompt.split()] - async with BufferedPublisher(stream, max_delay=timedelta(seconds=10)) as buffered: - buffered.publish("model-a") - buffered.publish("model-b") - offsets.append(await stream.get_offset()) +async def call_model(input: CallModelInput) -> list[str]: + stream = activity.client().get_stream(input.stream) + offsets = [str(await stream.append(word)) for word in input.prompt.split()] + async with BufferedAppender(stream, max_delay=timedelta(seconds=10)) as buffered: + buffered.append("model-a") + buffered.append("model-b") + offsets.append(str(await stream.get_offset())) return offsets @@ -81,36 +86,36 @@ def __init__(self, client: Client) -> None: self._client = client @activity.defn - def call_model_sync(self, input: SyncInput) -> int: - stream = SyncStream(self._client.to_stream(input.stream)) - last = stream.publish(*input.prompt.split()) - assert stream.get_offset() == last + 1 - return last + def call_model_sync(self, input: SyncInput) -> str: + stream = SyncStream(self._client.get_stream(input.stream)) + last = stream.append(*input.prompt.split()) + assert stream.get_offset() > last + return str(last) @activity.defn - def publishes_without_the_sync_stream(self, ref: streams.StreamRef[str]) -> str: + def appends_without_the_sync_stream(self, ref: streams.StreamRef[str]) -> str: try: - self._client.to_stream(ref).publish("x") - except streams.StreamUnsupportedError as err: - return type(err).__name__ + self._client.get_stream(ref).append("x") + except streams.StreamError as err: + return reason(err) return "wrote" @activity.defn def sync_read(self, ref: streams.StreamRef[str]) -> list[Any]: - stream = SyncStream(self._client.to_stream(ref)) + stream = SyncStream(self._client.get_stream(ref)) return [ item.data - for item in stream.subscribe(wait_new_records=False) + for item in stream.read(wait_new_records=False) if isinstance(item, streams.Record) ] @activity.defn async def read_back(ref: streams.StreamRef[str]) -> list[Any]: - stream = activity.client().to_stream(ref) + stream = activity.client().get_stream(ref) return [ item.data - async for item in stream.subscribe(wait_new_records=False) + async for item in stream.read(wait_new_records=False) if isinstance(item, streams.Record) ] @@ -119,7 +124,7 @@ async def read_back(ref: streams.StreamRef[str]) -> list[Any]: class Agent: @workflow.run async def run(self, prompt: str) -> list[Any]: - stream = await workflow.create_stream[str](own_id()) + stream = await workflow.create_stream(own_id(), payload_type=str) offsets = await workflow.execute_activity( call_model, CallModelInput(prompt, stream.ref), @@ -139,7 +144,7 @@ async def run(self, prompt: str) -> list[Any]: return [offsets, last, read] -async def test_activity_publishes_and_returns_offsets(stream_client: Client): +async def test_activity_appends_and_returns_offsets(stream_client: Client): async with new_worker( stream_client, Agent, @@ -158,9 +163,9 @@ async def test_activity_publishes_and_returns_offsets(stream_client: Client): data = records(items) written = ["hi", "there", "model-a", "model-b", "sync", "words"] assert [record.data for record in data] == written - assert offsets[:2] == [data[0].offset, data[1].offset] - assert offsets[2] == data[3].offset + 1 - assert last == data[5].offset + assert offsets[:2] == [str(data[0].offset), str(data[1].offset)] + assert streams.StreamOffset.parse(offsets[2]) > data[3].offset + assert streams.StreamOffset.parse(last) == data[5].offset assert read_tokens == written assert await closed(handle) @@ -178,7 +183,7 @@ def _executor() -> Any: async def flaky(input: CallModelInput) -> None: attempts[input.prompt] = attempts.get(input.prompt, 0) + 1 if attempts[input.prompt] == 1: - await activity.client().to_stream(input.stream).publish("attempt-1") + await activity.client().get_stream(input.stream).append("attempt-1") raise RuntimeError("first attempt fails") @@ -186,8 +191,8 @@ async def flaky(input: CallModelInput) -> None: class Retrying: @workflow.run async def run(self, key: str) -> None: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("start") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("start") await workflow.execute_activity( flaky, CallModelInput(key, stream.ref), @@ -223,7 +228,7 @@ async def reads_after_a_failed_attempt(input: FinishedStreams) -> list[Any]: attempts[input.key] = attempts.get(input.key, 0) + 1 if attempts[input.key] == 1: raise RuntimeError("first attempt fails") - items = activity.client().to_stream(input.closed).subscribe(wait_new_records=False) + items = activity.client().get_stream(input.closed).read(wait_new_records=False) return [item.data async for item in items if isinstance(item, streams.Record)] @@ -231,8 +236,8 @@ async def reads_after_a_failed_attempt(input: FinishedStreams) -> list[Any]: class ReadsFinishedStreams: @workflow.run async def run(self, key: str, expired: streams.StreamRef[str]) -> list[Any]: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("turn") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("turn") await stream.close() return await workflow.execute_activity( reads_after_a_failed_attempt, @@ -246,19 +251,25 @@ async def run(self, key: str, expired: streams.StreamRef[str]) -> list[Any]: ) -async def test_retry_goes_on_with_a_closed_or_expired_stream(stream_client: Client): +async def test_retry_goes_on_with_a_closed_or_expired_stream(client: Client): # On a Stream that accepts no write, the second attempt's open withdraws nothing, and the - # attempt only reads. - expired = await stream_client.create_stream[str]( - f"expired-{uuid.uuid4()}", retention=timedelta(milliseconds=200) + # attempt only reads. Retention is a backend setting, so the aged Stream expires while the + # closed one, created fresh in the run, is still within Retention. + config = client.config() + config["stream_backends"] = [ + MemoryStreamBackend(retention=timedelta(seconds=2), default=True) + ] + memory_client = Client(**config) + expired = await memory_client.create_stream( + f"expired-{uuid.uuid4()}", payload_type=str ) - await expired.publish("old") - await asyncio.sleep(0.5) + await expired.append("old") + await asyncio.sleep(3) key = str(uuid.uuid4()) async with new_worker( - stream_client, ReadsFinishedStreams, activities=[reads_after_a_failed_attempt] + memory_client, ReadsFinishedStreams, activities=[reads_after_a_failed_attempt] ) as worker: - result = await stream_client.execute_workflow( + result = await memory_client.execute_workflow( ReadsFinishedStreams.run, args=[key, expired.ref], id=f"wf-{uuid.uuid4()}", @@ -274,19 +285,19 @@ async def test_retry_goes_on_with_a_closed_or_expired_stream(stream_client: Clie @activity.defn async def fails_after_it_is_shown(input: CallModelInput) -> None: attempts[input.prompt] = attempts.get(input.prompt, 0) + 1 - tokens = activity.client().to_stream(input.stream) + tokens = activity.client().get_stream(input.stream) if attempts[input.prompt] == 1: - await tokens.publish("attempt-1") + await tokens.append("attempt-1") await shown[input.prompt].wait() raise RuntimeError("first attempt fails") - await tokens.publish("attempt-2") + await tokens.append("attempt-2") @workflow.defn class ShownThenWithdrawn: @workflow.run async def run(self, key: str) -> None: - stream = await workflow.create_stream[str](own_id()) + stream = await workflow.create_stream(own_id(), payload_type=str) await workflow.execute_activity( fails_after_it_is_shown, CallModelInput(key, stream.ref), @@ -310,9 +321,9 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) - stream = stream_client.get_stream[str](f"{handle.id}-tokens") - live = stream.subscribe(create_wait=timedelta(seconds=20)) - reconnects = stream.subscribe(create_wait=timedelta(seconds=20)) + stream = stream_client.get_stream(f"{handle.id}-tokens", payload_type=str) + live = stream.read(create_wait=timedelta(seconds=20)) + reconnects = stream.read(create_wait=timedelta(seconds=20)) first = await anext(live) before = await anext(reconnects) # The reader closes its connection and keeps only the Records that it showed and its @@ -321,7 +332,7 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( await reconnects.aclose() shown[key].set() live_items = [first, *[item async for item in live]] - resumed = [item async for item in stream.subscribe(offset=resume_at)] + resumed = [item async for item in stream.read(offset=resume_at)] assert isinstance(first, streams.Record) and first.data == "attempt-1" assert isinstance(before, streams.Record) and before.offset == first.offset [live_superseded] = [i for i in live_items if isinstance(i, streams.Superseded)] @@ -337,7 +348,7 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( @activity.defn async def wrong_backend(ref: streams.StreamRef[str]) -> str: try: - await activity.client().to_stream(ref).publish("x") + await activity.client().get_stream(ref).append("x") except streams.StreamConfigError: return "config" return "wrote" @@ -345,12 +356,12 @@ async def wrong_backend(ref: streams.StreamRef[str]) -> str: @activity.defn async def close_it(ref: streams.StreamRef[str]) -> str: - stream = activity.client().to_stream(ref) - await stream.publish("last") + stream = activity.client().get_stream(ref) + await stream.append("last") try: await stream.close() - except streams.StreamNotOwnerError as err: - return StreamErrorReason.Name(StreamErrorReason.ValueType(err.reason)) + except streams.StreamError as err: + return reason(err) return "closed" @@ -358,7 +369,7 @@ async def close_it(ref: streams.StreamRef[str]) -> str: class Closing: @workflow.run async def run(self) -> list[str]: - stream = await workflow.create_stream[str](own_id()) + stream = await workflow.create_stream(own_id(), payload_type=str) backend = await workflow.execute_activity( wrong_backend, streams.StreamRef[str](id=stream.ref.id, backend="elsewhere"), @@ -404,17 +415,13 @@ async def test_activity_checks_the_backend_and_the_owner(stream_client: Client): @activity.defn async def refused_calls(ref: streams.StreamRef[str]) -> list[str]: - stream = activity.client().to_stream(ref) + stream = activity.client().get_stream(ref) refused = [] try: async with stream: refused.append("entered") - except streams.StreamNotOwnerError as err: - refused.append(type(err).__name__) - try: - stream.producer("ingest", 1) - except streams.StreamUnsupportedError as err: - refused.append(type(err).__name__) + except streams.StreamError as err: + refused.append(reason(err)) return refused @@ -422,22 +429,22 @@ async def refused_calls(ref: streams.StreamRef[str]) -> list[str]: class RefusedCalls: @workflow.run async def run(self) -> list[str]: - stream = await workflow.create_stream[str](own_id()) + stream = await workflow.create_stream(own_id(), payload_type=str) refused = await workflow.execute_activity( refused_calls, stream.ref, start_to_close_timeout=timedelta(seconds=30) ) refused.append( await workflow.execute_activity_method( - SyncActivities.publishes_without_the_sync_stream, + SyncActivities.appends_without_the_sync_stream, stream.ref, start_to_close_timeout=timedelta(seconds=30), ) ) try: # Workflow code has no Stream, so this is for a caller that has no type checks. - BufferedPublisher(cast(Any, stream)) - except streams.StreamUnsupportedError as err: - refused.append(type(err).__name__) + BufferedAppender(cast(Any, stream)) + except streams.StreamError as err: + refused.append(reason(err)) await stream.close() return refused @@ -448,7 +455,7 @@ async def test_calls_that_an_activity_cant_make_raise(stream_client: Client): RefusedCalls, activities=[ refused_calls, - SyncActivities(stream_client).publishes_without_the_sync_stream, + SyncActivities(stream_client).appends_without_the_sync_stream, ], activity_executor=_executor(), ) as worker: @@ -456,8 +463,8 @@ async def test_calls_that_an_activity_cant_make_raise(stream_client: Client): RefusedCalls.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await handle.result() == [ - "StreamNotOwnerError", - *["StreamUnsupportedError"] * 3, + "STREAM_ERROR_REASON_NOT_OWNER", + *["STREAM_ERROR_REASON_UNSUPPORTED"] * 2, ] items = await read(handle) assert records(items) == [] @@ -466,18 +473,18 @@ async def test_calls_that_an_activity_cant_make_raise(stream_client: Client): @activity.defn async def leaves_a_failed_write(ref: streams.StreamRef[str]) -> str: # Nobody created the Stream, and no caller waits for the write. - activity.client().to_stream(ref).publish("lost") + activity.client().get_stream(ref).append("lost") return "returned" @activity.defn async def enters_a_client_stream(ref: streams.StreamRef[str]) -> str: - stream = activity.client().to_stream(ref) + stream = activity.client().get_stream(ref) try: async with stream: - await stream.publish("in the block") - except streams.StreamNotOwnerError as err: - return StreamErrorReason.Name(StreamErrorReason.ValueType(err.reason)) + await stream.append("in the block") + except streams.StreamError as err: + return reason(err) return "closed" @@ -508,7 +515,9 @@ async def run( async def test_attempt_end_raises_a_lost_write_and_a_client_stream_refuses_the_block( stream_client: Client, ): - stream = await stream_client.create_stream[str](f"client-{uuid.uuid4()}") + stream = await stream_client.create_stream( + f"client-{uuid.uuid4()}", payload_type=str + ) async with new_worker( stream_client, LostAndClientWrites, @@ -522,7 +531,7 @@ async def test_attempt_end_raises_a_lost_write_and_a_client_stream_refuses_the_b ) # An Activity closes no Stream, so the block fails before it runs, with no call. assert result == ["StreamNotFoundError", "STREAM_ERROR_REASON_NOT_OWNER"] - items = [item async for item in stream.subscribe(wait_new_records=False)] + items = [item async for item in stream.read(wait_new_records=False)] assert records(items) == [] @@ -535,66 +544,73 @@ def __init__(self, fail: bool = False) -> None: self.in_flight = 0 self.most_in_flight = 0 - def publish(self, *values: int, **options: Any) -> asyncio.Future[int]: + def append( + self, *values: int, **options: Any + ) -> asyncio.Future[streams.StreamOffset]: return asyncio.ensure_future(self._write(values)) - async def _write(self, values: Sequence[int]) -> int: + async def _write(self, values: Sequence[int]) -> streams.StreamOffset: if self.fail: - raise streams.StreamStorageUnavailableError("store down") + raise streams.StreamError( + "store down", + reason=StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE, + ) self.in_flight += 1 self.most_in_flight = max(self.most_in_flight, self.in_flight) await asyncio.sleep(0.01) self.in_flight -= 1 self.batches.append(tuple(values)) - return sum(len(batch) for batch in self.batches) + return streams.StreamOffset.parse( + str(sum(len(batch) for batch in self.batches)) + ) -async def test_buffered_publisher_batches_by_delay(): +async def test_buffered_appender_batches_by_delay(): stream = _Batches() - publisher = BufferedPublisher(stream, max_delay=timedelta(milliseconds=20)) - publisher.publish(1) - publisher.publish(2, 3) + appender = BufferedAppender(stream, max_delay=timedelta(milliseconds=20)) + appender.append(1) + appender.append(2, 3) await asyncio.sleep(0.1) assert stream.batches == [(1, 2, 3)] - publisher.publish(4) - assert await publisher.flush() == 4 + appender.append(4) + assert await appender.flush() == streams.StreamOffset.parse("4") assert stream.batches == [(1, 2, 3), (4,)] - assert await publisher.flush() == 4 + assert await appender.flush() == streams.StreamOffset.parse("4") -async def test_buffered_publisher_writes_a_full_batch_with_one_write_in_flight(): +async def test_buffered_appender_writes_a_full_batch_with_one_write_in_flight(): stream = _Batches() - publisher = BufferedPublisher(stream, max_delay=timedelta(minutes=10)) - publisher.publish(*range(1000)) - publisher.publish(*range(1000, 2500)) + appender = BufferedAppender(stream, max_delay=timedelta(minutes=10)) + appender.append(*range(1000)) + appender.append(*range(1000, 2500)) await asyncio.sleep(0.2) - # A full batch is written without waiting for the delay. The values published during its + # A full batch is written without waiting for the delay. The values appended during its # write are in the next batch. assert [len(batch) for batch in stream.batches] == [1000, 1500] assert stream.most_in_flight == 1 - assert await publisher.flush() == 2500 + assert await appender.flush() == streams.StreamOffset.parse("2500") -async def test_buffered_publisher_raises_a_failed_batch(): +async def test_buffered_appender_raises_a_failed_batch(): stream = _Batches(fail=True) - with pytest.raises(streams.StreamStorageUnavailableError): - async with BufferedPublisher(stream) as publisher: - publisher.publish(1) + with pytest.raises(streams.StreamError): + async with BufferedAppender(stream) as appender: + appender.append(1) with pytest.raises(TypeError): - BufferedPublisher(stream).publish() + BufferedAppender(stream).append() @activity.defn async def local_step(ref: streams.StreamRef[str]) -> list[Any]: # Each Workflow write is an awaited Local Activity, so the run's create and Records are in # the Store before a later Local Activity starts. - stream = activity.client().to_stream(ref) + stream = activity.client().get_stream(ref) seen = [ item.data - async for item in stream.subscribe(wait_new_records=False) + async for item in stream.read(wait_new_records=False) if isinstance(item, streams.Record) ] - await stream.publish("local") + await stream.append("local") return seen @@ -602,14 +618,14 @@ async def local_step(ref: streams.StreamRef[str]) -> list[Any]: class LocalOrder: @workflow.run async def run(self) -> list[Any]: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("before") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("before") seen = await workflow.execute_local_activity( local_step, stream.ref, start_to_close_timeout=timedelta(seconds=30), ) - await stream.publish("after") + await stream.append("after") await stream.close() return seen @@ -620,15 +636,15 @@ def __init__(self, other: Client) -> None: @activity.defn async def writes_with_both(self, ref: streams.StreamRef[str]) -> None: - await activity.client().to_stream(ref).publish("worker client") - await self._other.to_stream(ref).publish("other client") + await activity.client().get_stream(ref).append("worker client") + await self._other.get_stream(ref).append("other client") @workflow.defn class WritesWithTwoClients: @workflow.run async def run(self) -> None: - stream = await workflow.create_stream[str](own_id()) + stream = await workflow.create_stream(own_id(), payload_type=str) await workflow.execute_activity_method( TwoClients.writes_with_both, stream.ref, @@ -677,26 +693,26 @@ async def test_workflow_writes_go_out_before_a_local_activity(stream_client: Cli assert await closed(handle) -async def test_buffered_publisher_stays_failed(): +async def test_buffered_appender_stays_failed(): stream = _Batches(fail=True) - publisher = BufferedPublisher(stream) - publisher.publish(1, 2) - with pytest.raises(streams.StreamStorageUnavailableError): - await publisher.flush() + appender = BufferedAppender(stream) + appender.append(1, 2) + with pytest.raises(streams.StreamError): + await appender.flush() stream.fail = False - # A later batch would leave a gap, so the publisher refuses it and keeps the error. - with pytest.raises(streams.StreamStorageUnavailableError): - publisher.publish(3) - with pytest.raises(streams.StreamStorageUnavailableError): - await publisher.flush() - assert stream.batches == [] and publisher.unwritten == 2 + # A later batch would leave a gap, so the appender refuses it and keeps the error. + with pytest.raises(streams.StreamError): + appender.append(3) + with pytest.raises(streams.StreamError): + await appender.flush() + assert stream.batches == [] and appender.unwritten == 2 @activity.defn async def never_flushes(input: CallModelInput) -> None: - stream = activity.client().to_stream(input.stream) - publisher = BufferedPublisher(stream, max_delay=timedelta(minutes=10)) - publisher.publish(*input.prompt.split()) + stream = activity.client().get_stream(input.stream) + appender = BufferedAppender(stream, max_delay=timedelta(minutes=10)) + appender.append(*input.prompt.split()) @dataclass @@ -712,7 +728,7 @@ async def flaky_many(input: ManyStreams) -> None: assert input.maybe is not None if attempts[input.key] == 1: for ref in [*input.targets, input.maybe]: - await activity.client().to_stream(ref).publish("attempt-1") + await activity.client().get_stream(ref).append("attempt-1") raise RuntimeError("first attempt fails") @@ -720,8 +736,8 @@ async def flaky_many(input: ManyStreams) -> None: class Leftovers: @workflow.run async def run(self, key: str) -> list[Any]: - stream = await workflow.create_stream[str](own_id()) - other = await workflow.create_stream[str](own_id("other")) + stream = await workflow.create_stream(own_id(), payload_type=str) + other = await workflow.create_stream(own_id("other"), payload_type=str) await workflow.execute_activity( never_flushes, CallModelInput("a b", stream.ref), @@ -757,7 +773,7 @@ async def test_attempt_end_flushes_and_retry_finds_nested_streams( handle = await stream_client.start_workflow( Leftovers.run, key, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) - # The publisher was never flushed, so the end of the attempt wrote its values. + # The appender was never flushed, so the end of the attempt wrote its values. assert await handle.result() == ["a", "b", "attempt-1"] items = await read(handle) other = await read(handle, "other") @@ -921,7 +937,7 @@ class RetriesAfterContinueAsNew: @workflow.run async def run(self, key: str, ref: streams.StreamRef[str] | None) -> None: if ref is None: - created = await workflow.create_stream[str](own_id()) + created = await workflow.create_stream(own_id(), payload_type=str) workflow.continue_as_new(args=[key, created.ref]) await workflow.execute_activity( flaky, diff --git a/tests/contrib/streams/test_bindings.py b/tests/contrib/streams/test_bindings.py index 1c62df609d..24ef498696 100644 --- a/tests/contrib/streams/test_bindings.py +++ b/tests/contrib/streams/test_bindings.py @@ -67,7 +67,9 @@ async def test_a_poll_carries_the_next_offset_until_the_close(): ), models.ReadRecordsResponse(records=[], next_offset=10, done=True), ) - stream = FakeService(nexus_client).get_stream[str](id="wf:run") + stream = FakeService(nexus_client).get_stream( + models.StreamRef(id="wf:run"), body_type=str + ) items = [ item async for item in stream.iter_read_items( @@ -87,7 +89,9 @@ async def test_a_read_with_no_wait_ends_at_an_empty_page_that_didnt_move(): ), models.ReadRecordsResponse(records=[], next_offset=5), ) - stream = FakeService(nexus_client).get_stream[str](id="wf:run") + stream = FakeService(nexus_client).get_stream( + models.StreamRef(id="wf:run"), body_type=str + ) items = [ item async for item in stream.iter_read_items( @@ -105,7 +109,9 @@ async def test_a_poll_resumes_after_the_last_item_it_gave(): nexus_client = FakeNexusClient( models.ReadRecordsResponse(records=[_data(4), _data(7)], next_offset=8), ) - stream = FakeService(nexus_client).get_stream[str](id="wf:run") + stream = FakeService(nexus_client).get_stream( + models.StreamRef(id="wf:run"), body_type=str + ) items = stream.iter_read_items(start=models.StreamReadStart.StreamReadStartEnd) first = await anext(items) assert first.offset == 4 @@ -132,7 +138,9 @@ async def test_a_poll_resumes_at_a_superseded_item_and_keeps_its_record(): nexus_client = FakeNexusClient( models.ReadRecordsResponse(records=[superseded, _data(7)], next_offset=8), ) - stream = FakeService(nexus_client).get_stream[str](id="wf:run") + stream = FakeService(nexus_client).get_stream( + models.StreamRef(id="wf:run"), body_type=str + ) items = stream.iter_read_items() assert (await anext(items)).offset == 7 # The first Record of the batch has the Superseded item's offset and comes next, so a read diff --git a/tests/contrib/streams/test_facade.py b/tests/contrib/streams/test_facade.py index 8ce895627e..b12be9a8ec 100644 --- a/tests/contrib/streams/test_facade.py +++ b/tests/contrib/streams/test_facade.py @@ -73,14 +73,14 @@ def test_runtime_errors_map_to_stream_errors(): async def test_client_writes_reads_and_closes(client: Client): client, extension = fake_client(client) - stream = await client.create_stream[Turn]("chat-42") - first = await stream.publish(Turn("model")) - second = await stream.publish(Turn("user", "hello")) - assert second == first + 1 - assert await stream.get_offset() == second + 1 + stream = await client.create_stream("chat-42", payload_type=Turn) + first = await stream.append(Turn("model")) + second = await stream.append(Turn("user", "hello")) + assert first < second + assert await stream.get_offset() > second await stream.close() # A close writes no item. The read ends once it gave the last Record. - items = [item async for item in stream.subscribe()] + items = [item async for item in stream.read()] assert [item.kind for item in items] == [streams.ItemKind.RECORD] * 2 turn, token = items assert isinstance(turn, streams.Record) and turn.data == Turn("model") @@ -89,17 +89,21 @@ async def test_client_writes_reads_and_closes(client: Client): assert isinstance(turn.producer.epoch.kind, streams.ProducerEpochKindClient) assert extension.namespaces == {client.namespace} - resumed = [item async for item in stream.subscribe(offset=token.offset)] + resumed = [item async for item in stream.read(offset=token.offset)] assert [item.offset for item in resumed] == [token.offset] async def test_create_fails_on_an_id_that_exists(client: Client): client, extension = fake_client(client) - stream = await client.create_stream[str]("taken") + stream = await client.create_stream("taken", payload_type=str) with pytest.raises(streams.StreamAlreadyExistsError): - await client.create_stream[str]("taken") - # The create-or-get gets it, and ignores the options. - got = await client.create_or_get_stream[str]("taken", retention=timedelta(hours=1)) + await client.create_stream("taken", payload_type=str) + # USE_EXISTING gets the Stream instead of failing. + got = await client.create_stream( + "taken", + payload_type=str, + id_conflict_policy=streams.StreamIDConflictPolicy.USE_EXISTING, + ) assert got.ref == stream.ref policies = [create.id_conflict_policy for create in extension.creates] assert policies == [ @@ -114,7 +118,7 @@ async def test_create_fails_on_an_id_that_exists(client: Client): async def test_a_stream_gives_its_ref_and_its_wire_message(client: Client): client, _ = fake_client(client) - stream = await client.create_stream[Turn]("chat-proto") + stream = await client.create_stream("chat-proto", payload_type=Turn) assert stream.ref == streams.StreamRef(id="chat-proto", backend="fake") assert stream.proto_ref == temporalio.api.stream.v1.StreamRef( id="chat-proto", backend="fake" @@ -123,32 +127,32 @@ async def test_a_stream_gives_its_ref_and_its_wire_message(client: Client): async def test_read_waits_for_create(client: Client): client, _ = fake_client(client) - stream = client.get_stream[str]("tokens-7") + stream = client.get_stream("tokens-7", payload_type=str) with pytest.raises(streams.StreamNotFoundError): - async for _ in stream.subscribe(): + async for _ in stream.read(): pass async def create_later() -> None: - created = await client.create_stream[str](stream.ref.id) - await created.publish("five") + created = await client.create_stream(stream.ref.id, payload_type=str) + await created.append("five") await created.close() task = asyncio.create_task(create_later()) - items = [item async for item in stream.subscribe(create_wait=timedelta(seconds=5))] + items = [item async for item in stream.read(create_wait=timedelta(seconds=5))] await task assert [item.data for item in items if isinstance(item, streams.Record)] == ["five"] async def test_a_waiting_read_ends_at_a_close(client: Client): client, _ = fake_client(client) - stream = await client.create_stream[str]("closes-later") - reader = stream.subscribe() + stream = await client.create_stream("closes-later", payload_type=str) + reader = stream.read() async def read() -> list[str]: return [item.data async for item in reader if isinstance(item, streams.Record)] task = asyncio.create_task(read()) - await stream.publish("last") + await stream.append("last") await asyncio.sleep(0.05) await stream.close() assert await asyncio.wait_for(task, 5) == ["last"] @@ -160,8 +164,11 @@ def test_stream_errors_fail_the_workflow_not_the_task(): assert isinstance(err, ApplicationError) assert err.non_retryable and err.type == "StreamClosedError" assert err.reason == closed - # A Store outage is the one error that a retry can fix. - assert not streams.StreamStorageUnavailableError("down").non_retryable + # A Store outage is the one error that a retry can fix. Its reason, not a class, marks it. + outage = streams.StreamError( + "down", reason=StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE + ) + assert not outage.non_retryable mapped = _mapped( StreamCallError( int(RPCStatusCode.FAILED_PRECONDITION), "closed", _details(closed) @@ -182,11 +189,11 @@ def test_stream_errors_fail_the_workflow_not_the_task(): async def test_streams_of_one_client_share_a_producer_and_free_it(client: Client): client, extension = fake_client(client) - stream = await client.create_stream[str]("two-writers") - other = client.get_stream[str]("two-writers") - await stream.publish("1") - await other.publish("2") - items = [item async for item in stream.subscribe(wait_new_records=False)] + stream = await client.create_stream("two-writers", payload_type=str) + other = client.get_stream("two-writers", payload_type=str) + await stream.append("1") + await other.append("2") + items = [item async for item in stream.read(wait_new_records=False)] producers = { item.producer.id for item in items @@ -202,7 +209,7 @@ async def test_streams_of_one_client_share_a_producer_and_free_it(client: Client async def test_rpc_timeout_bounds_each_call_and_not_the_read(client: Client): client, extension = fake_client(client) - stream = await client.create_stream[str]("slow") + stream = await client.create_stream("slow", payload_type=str) call = extension.call async def slow_describe(method: str, input: bytes) -> bytes: @@ -215,16 +222,16 @@ async def slow_describe(method: str, input: bytes) -> bytes: await stream.get_offset(rpc_timeout=timedelta(milliseconds=100)) assert err.value.status == RPCStatusCode.DEADLINE_EXCEEDED - async def publish_later() -> None: + async def append_later() -> None: await asyncio.sleep(1) - await stream.publish("late") + await stream.append("late") # Each call of a waiting read returns before the deadline, so the read continues. - reader = stream.subscribe(rpc_timeout=timedelta(milliseconds=300)) - publisher = asyncio.create_task(publish_later()) + reader = stream.read(rpc_timeout=timedelta(milliseconds=300)) + appender = asyncio.create_task(append_later()) item = await anext(reader) await reader.aclose() - await publisher + await appender assert isinstance(item, streams.Record) and item.data == "late" @@ -248,9 +255,9 @@ async def test_codec_runs_once_for_each_write_and_page(client: Client): config = client.config() config["data_converter"] = DataConverter(payload_codec=codec) client, _ = fake_client(Client(**config)) - stream = await client.create_stream[str]("coded") - await stream.publish(*(str(n) for n in range(5))) - items = [item async for item in stream.subscribe(wait_new_records=False)] + stream = await client.create_stream("coded", payload_type=str) + await stream.append(*(str(n) for n in range(5))) + items = [item async for item in stream.read(wait_new_records=False)] assert [item.data for item in items if isinstance(item, streams.Record)] == [ str(n) for n in range(5) ] @@ -267,7 +274,7 @@ async def test_forked_process_gets_its_own_client_producers(client: Client): async def test_client_streams_are_of_the_exported_class(client: Client): client, _ = fake_client(client) - stream = client.get_stream[str]("exported") + stream = client.get_stream("exported", payload_type=str) assert isinstance(stream, temporalio.client.Stream) assert "Stream" in temporalio.client.__all__ @@ -283,10 +290,10 @@ def test_import_of_temporalio_client_loads_no_stream_code(): async def test_bad_create_and_read_arguments_raise_with_no_call(client: Client): client, extension = fake_client(client) with pytest.raises(ValueError, match="id is empty"): - await client.create_stream[str]("") - stream = client.get_stream[str]("args") + await client.create_stream("", payload_type=str) + stream = client.get_stream("args", payload_type=str) with pytest.raises(ValueError, match="from 1 to 1000"): - stream.subscribe(last=1001) + stream.read(last=1001) assert not extension.creates and not extension.namespaces @@ -296,7 +303,7 @@ class _Interrupt(BaseException): async def test_block_end_closes_on_any_exception(client: Client): client, extension = fake_client(client) - stream = await client.create_stream[str]("interrupted") + stream = await client.create_stream("interrupted", payload_type=str) with pytest.raises(_Interrupt): async with stream: raise _Interrupt @@ -305,8 +312,8 @@ async def test_block_end_closes_on_any_exception(client: Client): async def test_a_failed_close_is_tried_again(client: Client): client, extension = fake_client(client) - stream = await client.create_stream[str]("closes-twice") - await stream.publish("a") + stream = await client.create_stream("closes-twice", payload_type=str) + await stream.append("a") call = extension.call refused: list[str] = [] @@ -321,12 +328,15 @@ async def refuses_the_first_close(method: str, input: bytes) -> bytes: return await call(method, input) setattr(extension, "call", refuses_the_first_close) - with pytest.raises(streams.StreamStorageUnavailableError): + with pytest.raises(streams.StreamError) as failed: async with stream: pass + assert ( + failed.value.reason == StreamErrorReason.STREAM_ERROR_REASON_STORAGE_UNAVAILABLE + ) # The writer closed before the Store refused, so this object writes no more. with pytest.raises(streams.StreamClosedError, match="closed its writer"): - stream.publish("late") + stream.append("late") # The Stream is still open, and the next close goes through. assert not extension.streams["closes-twice"].closed await stream.close() diff --git a/tests/contrib/streams/test_pydantic.py b/tests/contrib/streams/test_pydantic.py index e301e6d699..2169792099 100644 --- a/tests/contrib/streams/test_pydantic.py +++ b/tests/contrib/streams/test_pydantic.py @@ -23,17 +23,17 @@ class TurnResult(pydantic.BaseModel): @activity.defn -async def call_model(input: CallModelInput) -> int: - stream = activity.client().to_stream(input.stream) - return await stream.publish(*input.prompt.split()) +async def call_model(input: CallModelInput) -> str: + stream = activity.client().get_stream(input.stream) + return str(await stream.append(*input.prompt.split())) @workflow.defn class TurnWorkflow: @workflow.run async def run(self, prompt: str) -> TurnResult: - stream = await workflow.create_stream[str]( - f"{workflow.info().workflow_id}-tokens" + stream = await workflow.create_stream( + f"{workflow.info().workflow_id}-tokens", payload_type=str ) await workflow.execute_activity( call_model, @@ -62,8 +62,8 @@ async def test_refs_in_models_round_trip_on_the_memory_store(client: Client): assert result.stream.owner is None assert result.stream.id == f"{handle.id}-tokens" assert result.stream.backend == "memory" - stream = client.to_stream(result.stream) - items = [item async for item in stream.subscribe()] + stream = client.get_stream(result.stream) + items = [item async for item in stream.read()] assert [i.data for i in items if isinstance(i, streams.Record)] == ["hi", "there"] status = await stream_status(client, result.stream.id) assert status == models.StreamStatus.StreamStatusClosed diff --git a/tests/contrib/streams/test_reader.py b/tests/contrib/streams/test_reader.py index 345bcab531..72d310b2f8 100644 --- a/tests/contrib/streams/test_reader.py +++ b/tests/contrib/streams/test_reader.py @@ -7,7 +7,7 @@ import temporalio.common import temporalio.converter -from temporalio.streams import StreamNotFoundError, _backend, _reader +from temporalio.streams import StreamNotFoundError, StreamOffset, _backend, _reader from temporalio.streams._items import Record, Superseded from temporalio.streams.system import models @@ -69,15 +69,17 @@ async def test_superseded_lists_each_withdrawn_offset_once(): async def items() -> AsyncGenerator[_reader.Item, None]: for offset, producer in written: - yield Record(data=None, offset=offset, producer=producer) - yield Superseded(offset=30, facts=facts) - yield Superseded(offset=31, facts=facts) + yield Record( + data=None, offset=StreamOffset._from_wire(offset), producer=producer + ) + yield Superseded(offset=StreamOffset._from_wire(30), facts=facts) + yield Superseded(offset=StreamOffset._from_wire(31), facts=facts) - read: _reader.Subscription[None] = _reader.Subscription( - items(), _reader.Position(0) - ) + read: _reader.Reader[None] = _reader.Reader(items(), _reader.Position(0)) superseded = [item async for item in read if isinstance(item, Superseded)] - assert [item.withdrawn_offsets for item in superseded] == [(10, 11, 12, 20), ()] + assert [ + tuple(str(offset) for offset in item.withdrawn_offsets) for item in superseded + ] == [("10", "11", "12", "20"), ()] class _Missing(_backend.StreamServiceCalls): diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 582e43730e..2811e9fd49 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -14,7 +14,7 @@ from temporalio.api.common.v1 import Payload from temporalio.api.enums.v1 import EventType, StreamErrorReason from temporalio.client import ( - BufferedPublisher, + BufferedAppender, Client, Interceptor, Stream, @@ -45,7 +45,7 @@ async def read_all(stream: Stream[Any]) -> list[Any]: - return [item async for item in stream.subscribe(create_wait=timedelta(seconds=20))] + return [item async for item in stream.read(create_wait=timedelta(seconds=20))] def records(items: list[Any]) -> list[streams.Record[Any]]: @@ -59,12 +59,12 @@ def records(items: list[Any]) -> list[streams.Record[Any]]: class PublishThenFailTask: @workflow.run async def run(self, key: str) -> None: - stream = await workflow.create_stream[str]( - f"{workflow.info().workflow_id}-tokens" + stream = await workflow.create_stream( + f"{workflow.info().workflow_id}-tokens", payload_type=str ) - await stream.publish("before") + await stream.append("before") await workflow.sleep(timedelta(milliseconds=10)) - await stream.publish("after") + await stream.append("after") if attempts.setdefault(key, 0) == 0: attempts[key] = 1 raise RuntimeError("fail this Workflow Task once") @@ -88,7 +88,7 @@ async def test_failed_workflow_task_is_withdrawn(stream_client: Client): ) await handle.result() stream_id = f"{handle.id}-tokens" - items = await read_all(stream_client.get_stream[str](stream_id)) + items = await read_all(stream_client.get_stream(stream_id, payload_type=str)) assert ( await stream_status(stream_client, stream_id) == models.StreamStatus.StreamStatusClosed @@ -109,15 +109,15 @@ class CallModelInput: @activity.defn async def local_tokens(input: CallModelInput) -> None: - await activity.client().to_stream(input.stream).publish(*input.prompt.split()) + await activity.client().get_stream(input.stream).append(*input.prompt.split()) @workflow.defn class LocalThenFailTask: @workflow.run async def run(self, key: str) -> None: - stream = await workflow.create_stream[str]( - f"{workflow.info().workflow_id}-tokens" + stream = await workflow.create_stream( + f"{workflow.info().workflow_id}-tokens", payload_type=str ) first = attempts.setdefault(key, 0) == 0 await workflow.execute_local_activity( @@ -154,7 +154,9 @@ async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Cli task_queue=worker.task_queue, ) await handle.result() - items = await read_all(stream_client.get_stream[str](f"{handle.id}-tokens")) + items = await read_all( + stream_client.get_stream(f"{handle.id}-tokens", payload_type=str) + ) written = records(items) superseded = [item for item in items if isinstance(item, streams.Superseded)] visible = [r for r in written if not any(i.withdraws(r) for i in superseded)] @@ -169,8 +171,8 @@ async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Cli class LocalOnlyInFailedTask: @workflow.run async def run(self, key: str) -> None: - stream = await workflow.create_stream[str]( - f"{workflow.info().workflow_id}-tokens" + stream = await workflow.create_stream( + f"{workflow.info().workflow_id}-tokens", payload_type=str ) if attempts.setdefault(key, 0) == 0: attempts[key] = 1 @@ -180,7 +182,7 @@ async def run(self, key: str) -> None: start_to_close_timeout=timedelta(seconds=30), ) raise RuntimeError("fail this Workflow Task once") - await stream.publish("other") + await stream.append("other") await stream.close() @@ -201,7 +203,7 @@ async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Cli ) await handle.result() stream_id = f"{handle.id}-tokens" - items = await read_all(stream_client.get_stream[str](stream_id)) + items = await read_all(stream_client.get_stream(stream_id, payload_type=str)) assert ( await stream_status(stream_client, stream_id) == models.StreamStatus.StreamStatusClosed @@ -236,9 +238,9 @@ async def closed(client: Client, handle: WorkflowHandle[Any, Any], name: str) -> @activity.defn async def call_model(input: CallModelInput) -> str: - stream = activity.client().to_stream(input.stream) + stream = activity.client().get_stream(input.stream) for word in input.prompt.split(): - await stream.publish(word) + await stream.append(word) return input.prompt.upper() @@ -246,15 +248,15 @@ async def call_model(input: CallModelInput) -> str: class Agent: @workflow.run async def run(self, prompt: str) -> str: - turns = await workflow.create_stream[Turn](own_id("turns")) - tokens = await workflow.create_stream[str](own_id("tokens")) - await turns.publish(Turn("model")) + turns = await workflow.create_stream(own_id("turns"), payload_type=Turn) + tokens = await workflow.create_stream(own_id("tokens"), payload_type=str) + await turns.append(Turn("model")) reply = await workflow.execute_activity( call_model, CallModelInput(prompt, tokens.ref), start_to_close_timeout=timedelta(seconds=30), ) - await turns.publish(Turn("done", reply)) + await turns.append(Turn("done", reply)) await turns.close() await tokens.close() return reply @@ -275,7 +277,7 @@ async def test_resume_after_a_superseded_item_keeps_its_record(stream_client: Cl ) await handle.result() stream = stream_of(stream_client, handle) - items = [item async for item in stream.subscribe(wait_new_records=False)] + items = [item async for item in stream.read(wait_new_records=False)] # The retried task's batch has its Superseded item and its first Record in one entry. record_offsets = {r.offset for r in records(items)} [shared] = [ @@ -283,7 +285,7 @@ async def test_resume_after_a_superseded_item_keeps_its_record(stream_client: Cl for i in items if isinstance(i, streams.Superseded) and i.offset in record_offsets ] - reader = stream.subscribe(wait_new_records=False) + reader = stream.read(wait_new_records=False) async for item in reader: if item == shared: break @@ -291,8 +293,7 @@ async def test_resume_after_a_superseded_item_keeps_its_record(stream_client: Cl await reader.aclose() assert resume_at == shared.offset resumed = [ - item - async for item in stream.subscribe(offset=resume_at, wait_new_records=False) + item async for item in stream.read(offset=resume_at, wait_new_records=False) ] assert [r.data for r in records(resumed) if r.offset == shared.offset] == ["after"] @@ -331,10 +332,10 @@ class WritesThenContinues: @workflow.run async def run(self, ref: streams.StreamRef[str] | None) -> str: if ref is None: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("first run") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("first run") workflow.continue_as_new(stream.ref) - await workflow.to_stream(ref).close() + await workflow.get_stream(ref).close() return "second run" @@ -356,8 +357,10 @@ async def test_later_run_closes_a_stream_it_did_not_write(stream_client: Client) class CancelledBlock: @workflow.run async def run(self) -> None: - async with await workflow.create_stream[str](own_id("scoped")) as turn: - await turn.publish("started") + async with await workflow.create_stream( + own_id("scoped"), payload_type=str + ) as turn: + await turn.append("started") await workflow.wait_condition(lambda: False) @@ -367,7 +370,7 @@ async def test_cancelled_block_closes_the_stream(stream_client: Client): CancelledBlock.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) turn = stream_of(stream_client, handle, "scoped") - reader = turn.subscribe(create_wait=timedelta(seconds=20)) + reader = turn.read(create_wait=timedelta(seconds=20)) assert isinstance(await anext(reader), streams.Record) await reader.aclose() await handle.cancel() @@ -380,10 +383,12 @@ async def test_cancelled_block_closes_the_stream(stream_client: Client): class PlainWriter: @workflow.run async def run(self) -> str: - stream = await workflow.create_stream[str](own_id("plain")) - await stream.publish("plain") - async with await workflow.create_stream[str](own_id("scoped")) as scoped: - await scoped.publish("scoped") + stream = await workflow.create_stream(own_id("plain"), payload_type=str) + await stream.append("plain") + async with await workflow.create_stream( + own_id("scoped"), payload_type=str + ) as scoped: + await scoped.append("scoped") await stream.close() return "done" @@ -442,13 +447,13 @@ async def test_records_go_through_the_payload_codec(stream_client: Client): Agent.run, "a b", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) await handle.result() - turns = codec_client.get_stream[Turn](f"{handle.id}-turns") - tokens = codec_client.get_stream[str](f"{handle.id}-tokens") + turns = codec_client.get_stream(f"{handle.id}-turns", payload_type=Turn) + tokens = codec_client.get_stream(f"{handle.id}-tokens", payload_type=str) turn_items = await read_all(turns) decoded = await read_all(tokens) # RawValue as the payload type reads the raw Payloads. - raw_tokens = codec_client.get_stream[RawValue](tokens.ref.id) - raw = [item async for item in raw_tokens.subscribe(wait_new_records=False)] + raw_tokens = codec_client.get_stream(tokens.ref.id, payload_type=RawValue) + raw = [item async for item in raw_tokens.read(wait_new_records=False)] assert [item.data for item in records(turn_items)] == [ Turn("model"), Turn("done", "A B"), @@ -466,12 +471,14 @@ def __init__(self) -> None: @workflow.run async def run(self) -> str: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("before restart") - async with await workflow.create_stream[str](own_id("scoped")) as scoped: - await scoped.publish("scoped") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("before restart") + async with await workflow.create_stream( + own_id("scoped"), payload_type=str + ) as scoped: + await scoped.append("scoped") await workflow.wait_condition(lambda: self.go) - await stream.publish("after restart") + await stream.append("after restart") await stream.close() return "done" @@ -487,7 +494,7 @@ async def test_new_worker_replays_and_writes_each_record_once(stream_client: Cli WaitsForSignal.run, id=f"wf-{uuid.uuid4()}", task_queue=task_queue ) first = stream_of(stream_client, handle) - reader = first.subscribe(create_wait=timedelta(seconds=20)) + reader = first.read(create_wait=timedelta(seconds=20)) assert isinstance(await anext(reader), streams.Record) await reader.aclose() # The new Worker has no cache, so it replays the run. A replay sends no create, which would @@ -524,13 +531,15 @@ async def test_backend_replays_a_history_made_without_it( async def test_backend_shutdown_after_client_writes(stream_client: Client): [backend] = stream_client.config()["stream_backends"] assert isinstance(backend, ExtensionStreamBackend) - stream = await stream_client.create_stream[str](f"shutdown-{uuid.uuid4()}") - async with BufferedPublisher(stream) as tokens: - tokens.publish("buffered") + stream = await stream_client.create_stream( + f"shutdown-{uuid.uuid4()}", payload_type=str + ) + async with BufferedAppender(stream) as tokens: + tokens.append("buffered") await backend.shutdown() written = [ item.data - async for item in stream.subscribe(wait_new_records=False) + async for item in stream.read(wait_new_records=False) if isinstance(item, streams.Record) ] assert written == ["buffered"] @@ -540,8 +549,8 @@ async def test_backend_shutdown_after_client_writes(stream_client: Client): class WaitsForever: @workflow.run async def run(self) -> None: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("waiting") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("waiting") await workflow.wait_condition(lambda: False) @@ -584,7 +593,7 @@ async def test_eviction_prints_nothing( handle = await stream_client.start_workflow( WaitsForever.run, id=f"wf-{uuid.uuid4()}", task_queue=w.task_queue ) - reader = stream_of(stream_client, handle).subscribe( + reader = stream_of(stream_client, handle).read( create_wait=timedelta(seconds=20) ) assert isinstance(await anext(reader), streams.Record) @@ -601,8 +610,8 @@ class AlwaysFails: async def run(self, non_retryable: bool) -> None: attempt = workflow.info().attempt # Each retry run sends the create of the first run, so it gets the same Stream. - stream = await workflow.create_stream[str](own_id()) - await stream.publish(f"run-{attempt}") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append(f"run-{attempt}") raise ApplicationError(f"attempt {attempt} fails", non_retryable=non_retryable) @@ -623,7 +632,7 @@ async def test_final_failure_under_a_retry_policy( with pytest.raises(WorkflowFailureError): await handle.result() stream = stream_of(stream_client, handle) - items = [item async for item in stream.subscribe(wait_new_records=False)] + items = [item async for item in stream.read(wait_new_records=False)] # The end of the chain doesn't close the Stream. assert not await closed(stream_client, handle, "tokens") written = records(items) @@ -642,12 +651,15 @@ async def test_final_failure_under_a_retry_policy( async def test_reader_stops_on_its_own_timer_and_resumes(stream_client: Client): - stream = await stream_client.create_stream[str](f"timer-{uuid.uuid4()}") - first = await stream.publish("one") + stream = await stream_client.create_stream( + f"timer-{uuid.uuid4()}", payload_type=str + ) + first = await stream.append("one") seen: list[streams.Record[Any]] = [] + reader = stream.read() async def read_until_quiet() -> None: - async for item in stream.subscribe(): + async for item in reader: assert isinstance(item, streams.Record) seen.append(item) @@ -655,12 +667,11 @@ async def read_until_quiet() -> None: with pytest.raises(asyncio.TimeoutError): await asyncio.wait_for(read_until_quiet(), timeout=0.5) assert [item.offset for item in seen] == [first] - await stream.publish("two") + resume_at = reader.next_offset + await stream.append("two") resumed = [ item.data - async for item in stream.subscribe( - offset=seen[-1].offset + 1, wait_new_records=False - ) + async for item in stream.read(offset=resume_at, wait_new_records=False) if isinstance(item, streams.Record) ] assert resumed == ["two"] @@ -676,16 +687,18 @@ async def test_two_backends_on_one_client(client: Client, redis_url: str): ] stream_client = Client(**config) stream_id = f"two-{uuid.uuid4()}" - on_redis = await stream_client.create_stream[str](stream_id) - on_memory = await stream_client.create_stream[str](stream_id, backend="memory") + on_redis = await stream_client.create_stream(stream_id, payload_type=str) + on_memory = await stream_client.create_stream( + stream_id, backend="memory", payload_type=str + ) assert (on_redis.ref.backend, on_memory.ref.backend) == ("redis", "memory") - await on_redis.publish("redis") - await on_memory.publish("memory") + await on_redis.append("redis") + await on_memory.append("memory") for backend, expected in (("redis", "redis"), ("memory", "memory")): - again = stream_client.get_stream[str](stream_id, backend=backend) + again = stream_client.get_stream(stream_id, backend=backend, payload_type=str) data = [ item.data - async for item in again.subscribe(wait_new_records=False) + async for item in again.read(wait_new_records=False) if isinstance(item, streams.Record) ] assert data == [expected] @@ -717,8 +730,8 @@ async def execute_workflow(self, input: ExecuteWorkflowInput) -> Any: class PublishesClosesAndReturns: @workflow.run async def run(self) -> str: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("token") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("token") await stream.close() return "done" @@ -764,7 +777,7 @@ async def test_close_survives_a_failed_last_task(stream_client: Client, cache: i ) assert await handle.result() == "done" stream = stream_of(client, handle) - items = [item async for item in stream.subscribe(wait_new_records=False)] + items = [item async for item in stream.read(wait_new_records=False)] assert await closed(client, handle, "tokens") assert [r.data for r in records(items)] == ["token"] assert await close_activity_events(handle) == [ @@ -779,14 +792,16 @@ class WritesToStream: @workflow.run async def run(self, ref: streams.StreamRef[str]) -> str: try: - await workflow.to_stream(ref).publish("from workflow") + await workflow.get_stream(ref).append("from workflow") except streams.StreamError as err: return StreamErrorReason.Name(StreamErrorReason.ValueType(err.reason)) return "wrote" async def test_write_to_a_closed_stream_raises_at_the_await(stream_client: Client): - stream = await stream_client.create_stream[str](f"closed-{uuid.uuid4()}") + stream = await stream_client.create_stream( + f"closed-{uuid.uuid4()}", payload_type=str + ) await stream.close() async with new_worker(stream_client, WritesToStream) as worker: assert ( @@ -799,7 +814,7 @@ async def test_write_to_a_closed_stream_raises_at_the_await(stream_client: Clien == "STREAM_ERROR_REASON_CLOSED" ) # The close wrote no item, so a read of the closed Stream ends at once with none. - assert [item async for item in stream.subscribe()] == [] + assert [item async for item in stream.read()] == [] @workflow.defn @@ -807,12 +822,12 @@ class ContinuesThenRetries: @workflow.run async def run(self, ref: streams.StreamRef[str] | None) -> str: if ref is None: - created = await workflow.create_stream[str](own_id()) - await created.publish("first") + created = await workflow.create_stream(own_id(), payload_type=str) + await created.append("first") workflow.continue_as_new(created.ref) - stream = workflow.to_stream(ref) + stream = workflow.get_stream(ref) info = workflow.info() - await stream.publish(f"attempt-{info.attempt}") + await stream.append(f"attempt-{info.attempt}") if info.attempt == 1: raise ApplicationError("the continued run fails once") await stream.close() diff --git a/tests/contrib/streams/test_refs.py b/tests/contrib/streams/test_refs.py index 0ce3fe40de..a35a8c9e71 100644 --- a/tests/contrib/streams/test_refs.py +++ b/tests/contrib/streams/test_refs.py @@ -143,9 +143,13 @@ def _producer(id: str, attempt: int) -> streams.Producer: def test_read_items_have_a_kind_that_narrows_their_type(): - record = streams.Record(data=Turn("model"), offset=3, producer=_producer("a", 1)) + record = streams.Record( + data=Turn("model"), + offset=streams.StreamOffset.parse("3"), + producer=_producer("a", 1), + ) superseded = streams.Superseded( - offset=4, + offset=streams.StreamOffset.parse("4"), facts=streams.StreamSuperseded( writer=streams.StreamWriter(producer=_producer("a", 2), failed_tasks=[]) ), @@ -165,7 +169,7 @@ def test_read_items_have_a_kind_that_narrows_their_type(): def test_superseded_withdraws_by_the_producer_of_an_earlier_record(): superseded = streams.Superseded( - offset=9, + offset=streams.StreamOffset.parse("9"), facts=streams.StreamSuperseded( writer=streams.StreamWriter(producer=_producer("a", 2), failed_tasks=[]) ), diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index ee4994452e..5fb0d4ccb3 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -9,7 +9,7 @@ from temporalio import streams, workflow from temporalio.api.enums.v1 import StreamErrorReason -from temporalio.client import BufferedPublisher, Client, Stream +from temporalio.client import BufferedAppender, Client, Stream from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.converter import DataConverter from temporalio.streams.system import models @@ -24,42 +24,45 @@ class Turn: text: str = "" +_BEGINNING = streams.StreamOffset.parse("0") + + async def test_unknown_backend_raises_before_any_call(client: Client): client, extension = fake_client(client) with pytest.raises(streams.StreamConfigError, match="no Stream backend redis"): - await client.create_stream[Turn]("chat-1", backend="redis") + await client.create_stream("chat-1", payload_type=Turn, backend="redis") with pytest.raises(streams.StreamConfigError, match="It has fake"): - client.get_stream[Turn]("chat-1", backend="redis") + client.get_stream("chat-1", payload_type=Turn, backend="redis") with pytest.raises(streams.StreamConfigError, match="It has fake"): - client.to_stream(streams.StreamRef(id="chat-1", backend="redis")) + client.get_stream(streams.StreamRef(id="chat-1", backend="redis")) assert not extension.streams and not extension.namespaces - stream = await client.create_stream[Turn]("chat-1") + stream = await client.create_stream("chat-1", payload_type=Turn) assert stream.ref.backend == "fake" async def test_client_without_backend_has_no_transport(client: Client): with pytest.raises(streams.StreamConfigError, match="no Stream backend"): - client.get_stream[Turn]("chat-1") + client.get_stream("chat-1", payload_type=Turn) with pytest.raises(streams.StreamConfigError, match="no Stream backend"): - await client.create_stream[Turn]("chat-1") + await client.create_stream("chat-1", payload_type=Turn) -async def test_to_stream_gives_the_stream_of_a_ref(client: Client): +async def test_get_stream_gives_the_stream_of_a_ref(client: Client): client, _ = fake_client(client) - stream = await client.create_stream[Turn]("chat-2") + stream = await client.create_stream("chat-2", payload_type=Turn) converter = DataConverter.default.payload_converter ref = converter.from_payload( converter.to_payload(stream.ref), streams.StreamRef[Turn] ) assert ref == stream.ref - typed = client.to_stream(ref, payload_type=Turn) - assert typed.ref is ref and await typed.get_offset() == 0 - await typed.publish(Turn("model", "hi")) - [record] = [r async for r in typed.subscribe(wait_new_records=False)] + typed = client.get_stream(ref.id, payload_type=Turn, backend=ref.backend) + assert typed.ref == ref and await typed.get_offset() == _BEGINNING + await typed.append(Turn("model", "hi")) + [record] = [r async for r in typed.read(wait_new_records=False)] assert isinstance(record, streams.Record) and record.data == Turn("model", "hi") # With no payload type, a read decodes each Record as a JSON value. - untyped = client.to_stream(ref) - [raw] = [r async for r in untyped.subscribe(wait_new_records=False)] + untyped = client.get_stream(ref) + [raw] = [r async for r in untyped.read(wait_new_records=False)] assert isinstance(raw, streams.Record) and raw.data == { "kind": "model", "text": "hi", @@ -68,22 +71,22 @@ async def test_to_stream_gives_the_stream_of_a_ref(client: Client): async def test_a_call_with_no_type_argument_reads_json_values(client: Client): client, _ = fake_client(client) - typed = await client.create_stream[Turn]("chat-3") - await typed.publish(Turn("model", "hi")) + typed = await client.create_stream("chat-3", payload_type=Turn) + await typed.append(Turn("model", "hi")) untyped: Any = client.get_stream("chat-3") - [raw] = [r async for r in untyped.subscribe(wait_new_records=False)] + [raw] = [r async for r in untyped.read(wait_new_records=False)] assert raw.data == {"kind": "model", "text": "hi"} -async def test_publish_starts_the_write_at_the_call(client: Client): +async def test_append_starts_the_write_at_the_call(client: Client): client, _ = fake_client(client) - stream = await client.create_stream[str]("ordered") - futures = [stream.publish(str(n)) for n in range(3)] + stream = await client.create_stream("ordered", payload_type=str) + futures = [stream.append(str(n)) for n in range(3)] # A cancel of a future doesn't stop its write, which already has its place. futures[1].cancel() - stream.publish("3") + stream.append("3") await stream.close() - items = [item async for item in stream.subscribe()] + items = [item async for item in stream.read()] assert [item.data for item in items if isinstance(item, streams.Record)] == [ "0", "1", @@ -92,34 +95,35 @@ async def test_publish_starts_the_write_at_the_call(client: Client): ] assert [await futures[0], await futures[2]] == [items[0].offset, items[2].offset] with pytest.raises(streams.StreamClosedError): - stream.publish("4") + stream.append("4") async def test_next_call_raises_a_write_error_that_no_caller_got(client: Client): client, _ = fake_client(client) - stream = await client.create_stream[str]("lost") - await client.get_stream[str]("lost").close() - late = stream.publish("late") + stream = await client.create_stream("lost", payload_type=str) + await client.get_stream("lost", payload_type=str).close() + late = stream.append("late") await asyncio.wait([late]) with pytest.raises(streams.StreamClosedError): await stream.get_offset() # The call owns the error, so the next one makes its call. - assert await stream.get_offset() == 0 + assert await stream.get_offset() == _BEGINNING async def test_a_client_cant_close_a_stream_bound_to_a_workflow(client: Client): client, extension = fake_client(client) - await client.create_stream[str]("bound") + await client.create_stream("bound", payload_type=str) owner = streams.Execution( type=streams.ExecutionType.ExecutionTypeWorkflow, business_id="wf", run_id="run", ) ref: streams.StreamRef[str] = streams.StreamRef(id="bound", owner=owner) - bound = client.to_stream(ref) - with pytest.raises(streams.StreamNotOwnerError): + bound = client.get_stream(ref) + with pytest.raises(streams.StreamError) as not_owner: await bound.close() - with pytest.raises(streams.StreamNotOwnerError): + assert not_owner.value.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_OWNER + with pytest.raises(streams.StreamError): async with bound: pass assert not extension.streams["bound"].closed @@ -139,7 +143,7 @@ def stream(self) -> dict[str, str]: async def test_query_and_workflow_results_give_refs(client: Client): client, _ = fake_client(client) - created = await client.create_stream[str]("handed-out") + created = await client.create_stream("handed-out", payload_type=str) async with new_worker(client, HandsOutStream) as worker: # By name, since the method's own return type is the JSON. handle = await client.start_workflow( @@ -152,9 +156,9 @@ async def test_query_and_workflow_results_give_refs(client: Client): queried = await handle.query("stream", result_type=streams.StreamRef[str]) # The ref has no backend, so its Stream uses the default one. assert queried.id == "handed-out" and queried.backend is None - await client.to_stream(queried, payload_type=str).publish("from the query result") - await client.to_stream(result).publish("from the Workflow result") - items = [item async for item in created.subscribe(wait_new_records=False)] + await client.get_stream(queried).append("from the query result") + await client.get_stream(result).append("from the Workflow result") + items = [item async for item in created.read(wait_new_records=False)] assert [item.data for item in items if isinstance(item, streams.Record)] == [ "from the query result", "from the Workflow result", @@ -162,53 +166,68 @@ async def test_query_and_workflow_results_give_refs(client: Client): async def test_async_with_closes_a_client_stream(stream_client: Client): - async with await stream_client.create_stream[str](f"block-{uuid.uuid4()}") as done: - await done.publish("a") - failed = await stream_client.create_stream[str](f"block-{uuid.uuid4()}") + async with await stream_client.create_stream( + f"block-{uuid.uuid4()}", payload_type=str + ) as done: + await done.append("a") + failed = await stream_client.create_stream( + f"block-{uuid.uuid4()}", payload_type=str + ) with pytest.raises(ValueError): async with failed: raise ValueError("turn failed") - early = await stream_client.create_stream[str](f"block-{uuid.uuid4()}") + early = await stream_client.create_stream(f"block-{uuid.uuid4()}", payload_type=str) async with early: await early.close() - cancelled = await stream_client.create_stream[str](f"block-{uuid.uuid4()}") + cancelled = await stream_client.create_stream( + f"block-{uuid.uuid4()}", payload_type=str + ) with pytest.raises(asyncio.CancelledError): async with cancelled: raise asyncio.CancelledError() for stream in (done, failed, early, cancelled): # A read of a closed Stream ends once it gave the last item, with no wait. items = await asyncio.wait_for( - _records(stream_client.get_stream[str](stream.ref.id)), 5 + _records(stream_client.get_stream(stream.ref.id, payload_type=str)), 5 ) assert items == (["a"] if stream is done else []) with pytest.raises(streams.StreamClosedError): - await stream_client.get_stream[str](stream.ref.id).publish("late") + await stream_client.get_stream(stream.ref.id, payload_type=str).append( + "late" + ) async def _records(stream: Stream[Any]) -> list[Any]: - return [r.data async for r in stream.subscribe() if isinstance(r, streams.Record)] + return [r.data async for r in stream.read() if isinstance(r, streams.Record)] async def test_create_fails_on_a_stream_that_exists(stream_client: Client): stream_id = f"again-{uuid.uuid4()}" - first = await stream_client.create_stream[str]( - stream_id, retention=timedelta(days=1) - ) - await first.publish("a") + first = await stream_client.create_stream(stream_id, payload_type=str) + await first.append("a") with pytest.raises(streams.StreamAlreadyExistsError) as exists: - await stream_client.create_stream[str](stream_id) + await stream_client.create_stream(stream_id, payload_type=str) assert exists.value.reason == StreamErrorReason.STREAM_ERROR_REASON_ALREADY_EXISTS - # The create-or-get ignores the other Retention and returns the Stream. - again = await stream_client.create_or_get_stream[str]( - stream_id, retention=timedelta(days=2) + # USE_EXISTING returns the Stream instead of failing. + again = await stream_client.create_stream( + stream_id, + payload_type=str, + id_conflict_policy=streams.StreamIDConflictPolicy.USE_EXISTING, ) assert again.ref == first.ref - assert await again.get_offset() == await first.get_offset() > 0 + again_offset = await again.get_offset() + assert again_offset == await first.get_offset() + assert again_offset > _BEGINNING await first.close() - # A closed Stream fails both creates with CLOSED, which comes before ALREADY_EXISTS. - for create in (stream_client.create_stream, stream_client.create_or_get_stream): + # A closed Stream fails both policies with CLOSED, which comes before ALREADY_EXISTS. + for policy in ( + streams.StreamIDConflictPolicy.FAIL, + streams.StreamIDConflictPolicy.USE_EXISTING, + ): with pytest.raises(streams.StreamClosedError) as closed: - await create[str](stream_id) + await stream_client.create_stream( + stream_id, payload_type=str, id_conflict_policy=policy + ) assert closed.value.reason == StreamErrorReason.STREAM_ERROR_REASON_CLOSED @@ -227,59 +246,68 @@ async def test_a_create_retry_with_the_same_request_id_succeeds(stream_client: C ) -async def test_publish_to_a_stream_that_nobody_created_raises(stream_client: Client): - stream = stream_client.get_stream[str](f"missing-{uuid.uuid4()}") +async def test_append_to_a_stream_that_nobody_created_raises(stream_client: Client): + stream = stream_client.get_stream(f"missing-{uuid.uuid4()}", payload_type=str) with pytest.raises(streams.StreamNotFoundError) as missing: - await stream.publish("lost") + await stream.append("lost") assert missing.value.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND async def test_next_offset_resumes_after_the_last_item(stream_client: Client): - stream = await stream_client.create_stream[str](f"resume-{uuid.uuid4()}") - last = await stream.publish("a", "b") - tokens = stream.subscribe(wait_new_records=False) - assert tokens.next_offset == 0 + stream = await stream_client.create_stream( + f"resume-{uuid.uuid4()}", payload_type=str + ) + last = await stream.append("a", "b") + tokens = stream.read(wait_new_records=False) + assert tokens.next_offset == _BEGINNING assert [item.data async for item in tokens if isinstance(item, streams.Record)] == [ "a", "b", ] - assert tokens.next_offset == last + 1 - new = stream.subscribe(offset=streams.END, wait_new_records=False) + assert tokens.next_offset is not None and tokens.next_offset > last + resume_from = tokens.next_offset + new = stream.read(offset=streams.END, wait_new_records=False) assert new.next_offset is None assert [item async for item in new] == [] - assert new.next_offset == last + 1 - await stream.publish("c") - resumed = stream.subscribe(offset=tokens.next_offset, wait_new_records=False) + assert new.next_offset is not None and new.next_offset > last + await stream.append("c") + resumed = stream.read(offset=resume_from, wait_new_records=False) assert [ item.data async for item in resumed if isinstance(item, streams.Record) ] == ["c"] async def test_a_waiting_read_ends_when_the_stream_closes(stream_client: Client): - stream = await stream_client.create_stream[str](f"wakes-{uuid.uuid4()}") - await stream.publish("a") - reader = asyncio.create_task(_records(stream_client.get_stream[str](stream.ref.id))) + stream = await stream_client.create_stream( + f"wakes-{uuid.uuid4()}", payload_type=str + ) + await stream.append("a") + reader = asyncio.create_task( + _records(stream_client.get_stream(stream.ref.id, payload_type=str)) + ) await asyncio.sleep(0.2) assert not reader.done() await stream.close() assert await asyncio.wait_for(reader, 10) == ["a"] # A read from the end of a closed Stream has no item left, so it answers at once. - tail = stream_client.get_stream[str](stream.ref.id).subscribe(offset=streams.END) + tail = stream_client.get_stream(stream.ref.id, payload_type=str).read( + offset=streams.END + ) assert await asyncio.wait_for(_items(tail), 5) == [] -async def _items(read: streams.Subscription[Any]) -> list[Any]: +async def _items(read: streams.Reader[Any]) -> list[Any]: return [item async for item in read] async def test_last_reads_the_newest_records_then_follows(stream_client: Client): - stream = await stream_client.create_stream[str](f"last-{uuid.uuid4()}") - await stream.publish("a", "b", "c") + stream = await stream_client.create_stream(f"last-{uuid.uuid4()}", payload_type=str) + await stream.append("a", "b", "c") # A chat view shows the last few Records first, then waits for new ones. - tail = stream.subscribe(last=2) + tail = stream.read(last=2) assert tail.next_offset is None shown = [await anext(tail), await anext(tail)] - await stream.publish("d") + await stream.append("d") shown.append(await anext(tail)) await tail.aclose() assert [item.data for item in shown if isinstance(item, streams.Record)] == [ @@ -287,38 +315,42 @@ async def test_last_reads_the_newest_records_then_follows(stream_client: Client) "c", "d", ] - newest = stream.subscribe(last=1, wait_new_records=False) + newest = stream.read(last=1, wait_new_records=False) assert [item.data async for item in newest if isinstance(item, streams.Record)] == [ "d" ] with pytest.raises(ValueError, match="no offset"): - stream.subscribe(last=2, offset=0) + stream.read(last=2, offset=_BEGINNING) async def test_payload_type_of_the_call_decodes_each_value(stream_client: Client): - stream = await stream_client.create_stream[Turn](f"typed-{uuid.uuid4()}") - await stream.publish(Turn("model")) - turns = stream_client.get_stream[Turn](stream.ref.id) - dicts = stream_client.get_stream[dict[str, Any]](stream.ref.id) + stream = await stream_client.create_stream( + f"typed-{uuid.uuid4()}", payload_type=Turn + ) + await stream.append(Turn("model")) + turns = stream_client.get_stream(stream.ref.id, payload_type=Turn) + dicts = stream_client.get_stream(stream.ref.id, payload_type=dict[str, Any]) for reader, expected in ( (turns, Turn("model")), (dicts, {"kind": "model", "text": ""}), ): - read = reader.subscribe(wait_new_records=False) + read = reader.read(wait_new_records=False) assert [r.data async for r in read if isinstance(r, streams.Record)] == [ expected ] -async def test_buffered_publisher_keeps_the_publish_order(stream_client: Client): - stream = await stream_client.create_stream[Turn](f"buffered-{uuid.uuid4()}") - async with BufferedPublisher(stream) as publisher: - publisher.publish(Turn("user")) - publisher.publish(Turn("a"), Turn("b")) - publisher.publish(Turn("model")) +async def test_buffered_appender_keeps_the_append_order(stream_client: Client): + stream = await stream_client.create_stream( + f"buffered-{uuid.uuid4()}", payload_type=Turn + ) + async with BufferedAppender(stream) as appender: + appender.append(Turn("user")) + appender.append(Turn("a"), Turn("b")) + appender.append(Turn("model")) written = [ item.data - async for item in stream.subscribe(wait_new_records=False) + async for item in stream.read(wait_new_records=False) if isinstance(item, streams.Record) ] assert written == [Turn("user"), Turn("a"), Turn("b"), Turn("model")] @@ -328,15 +360,15 @@ async def test_record_that_fails_to_decode_raises_and_resumes_past_it( stream_client: Client, ): stream_id = f"poison-{uuid.uuid4()}" - writer = await stream_client.create_stream[str](stream_id) - bad = await writer.publish("not a turn") - reader = stream_client.get_stream[Turn](stream_id) - await reader.publish(Turn("after")) - read = reader.subscribe(wait_new_records=False) + writer = await stream_client.create_stream(stream_id, payload_type=str) + bad = await writer.append("not a turn") + reader = stream_client.get_stream(stream_id, payload_type=Turn) + await reader.append(Turn("after")) + read = reader.read(wait_new_records=False) with pytest.raises(streams.StreamError, match=f"offset {bad}"): await anext(read) - assert read.next_offset == bad + 1 - resumed = reader.subscribe(offset=read.next_offset, wait_new_records=False) + assert read.next_offset is not None and read.next_offset > bad + resumed = reader.read(offset=read.next_offset, wait_new_records=False) assert [ item.data async for item in resumed if isinstance(item, streams.Record) ] == [Turn("after")] @@ -344,19 +376,21 @@ async def test_record_that_fails_to_decode_raises_and_resumes_past_it( async def test_record_over_the_store_limit_raises_too_large(stream_client: Client): # The client's writer splits a batch at the Store's limit, so only one Record can be over it. - stream = await stream_client.create_stream[str](f"large-{uuid.uuid4()}") - with pytest.raises(streams.StreamTooLargeError) as refused: - await stream.publish("x" * 2_000_000) + stream = await stream_client.create_stream( + f"large-{uuid.uuid4()}", payload_type=str + ) + with pytest.raises(streams.StreamError) as refused: + await stream.append("x" * 2_000_000) assert refused.value.reason == StreamErrorReason.STREAM_ERROR_REASON_TOO_LARGE assert refused.value.non_retryable - assert await stream.get_offset() == 0 + assert await stream.get_offset() == _BEGINNING async def test_read_keeps_no_offsets_of_client_records(stream_client: Client): # Nothing withdraws a client's Records, so a long UI read of them keeps no state. - stream = await stream_client.create_stream[str](f"chat-{uuid.uuid4()}") - await stream.publish("a", "b", "c") - read = stream.subscribe(wait_new_records=False) + stream = await stream_client.create_stream(f"chat-{uuid.uuid4()}", payload_type=str) + await stream.append("a", "b", "c") + read = stream.read(wait_new_records=False) assert [item.data async for item in read if isinstance(item, streams.Record)] == [ "a", "b", @@ -365,32 +399,46 @@ async def test_read_keeps_no_offsets_of_client_records(stream_client: Client): assert read._shown == {} -async def test_create_of_an_expired_stream_raises(stream_client: Client): - # An open standalone Stream expires one Retention after its last write. +async def test_create_of_an_expired_stream_raises(client: Client): + # Retention is a backend setting now. An open standalone Stream expires one Retention + # after its last write. + config = client.config() + config["stream_backends"] = [ + MemoryStreamBackend(retention=timedelta(milliseconds=200), default=True) + ] + one = Client(**config) stream_id = f"expired-{uuid.uuid4()}" - retention = timedelta(milliseconds=200) - stream = await stream_client.create_stream[str](stream_id, retention=retention) - await stream.publish("old") + stream = await one.create_stream(stream_id, payload_type=str) + await stream.append("old") await asyncio.sleep(0.5) - with pytest.raises(streams.StreamExpiredError): - await stream_client.create_or_get_stream[str](stream_id, retention=retention) + with pytest.raises(streams.StreamError) as expired: + await one.create_stream( + stream_id, + payload_type=str, + id_conflict_policy=streams.StreamIDConflictPolicy.USE_EXISTING, + ) + assert expired.value.reason == StreamErrorReason.STREAM_ERROR_REASON_EXPIRED async def test_named_producer_fences_its_earlier_epoch(stream_client: Client): - stream = await stream_client.create_stream[str](f"named-{uuid.uuid4()}") + stream = await stream_client.create_stream( + f"named-{uuid.uuid4()}", payload_type=str + ) # A server that restarts writes as the same name with a higher epoch, so the zombie that it # replaces can't write after it. - before = stream.producer("ingest", epoch=1) - await before.publish("before restart") - after = stream.producer("ingest", epoch=2) - await after.publish("after restart") - with pytest.raises(streams.StreamFencedError): - await before.publish("zombie") + with streams.producer("ingest", epoch=1): + await stream.append("before restart") + with streams.producer("ingest", epoch=2): + await stream.append("after restart") + with streams.producer("ingest", epoch=1): + with pytest.raises(streams.StreamFencedError): + await stream.append("zombie") # The earlier epoch's Records are real messages, so nothing withdraws them. - items = [item async for item in stream.subscribe(wait_new_records=False)] + items = [item async for item in stream.read(wait_new_records=False)] assert [item.kind for item in items] == [streams.ItemKind.RECORD] * 2 with pytest.raises(ValueError): - stream.producer("ingest", epoch=0) + with streams.producer("ingest", epoch=0): + pass async def test_clients_of_one_runtime_share_the_backend_calls(stream_client: Client): @@ -398,10 +446,10 @@ async def test_clients_of_one_runtime_share_the_backend_calls(stream_client: Cli other = Client(**stream_client.config()) [backend] = stream_client.config(active_config=True)["stream_backends"] assert backend._backend_for(other) is backend._backend_for(stream_client) - stream = await other.create_stream[str](f"shared-{uuid.uuid4()}") - await stream.publish("from the other client") - again = stream_client.get_stream[str](stream.ref.id) - items = [item async for item in again.subscribe(wait_new_records=False)] + stream = await other.create_stream(f"shared-{uuid.uuid4()}", payload_type=str) + await stream.append("from the other client") + again = stream_client.get_stream(stream.ref.id, payload_type=str) + items = [item async for item in again.read(wait_new_records=False)] assert [item.data for item in items if isinstance(item, streams.Record)] == [ "from the other client" ] @@ -412,14 +460,14 @@ async def test_default_is_native_with_no_default_backend(client: Client): config["stream_backends"] = [MemoryStreamBackend()] one = Client(**config) with pytest.raises(streams.StreamConfigError, match="default is native"): - one.get_stream[str]("d-2") + one.get_stream("d-2", payload_type=str) with pytest.raises(streams.StreamConfigError, match="default is native"): - await one.create_stream[str]("d-2") + await one.create_stream("d-2", payload_type=str) with pytest.raises(streams.StreamConfigError, match="default is native"): - one.to_stream(streams.StreamRef(id="d-2")) - on_memory = one.get_stream[str]("d-2", backend="memory") + one.get_stream(streams.StreamRef(id="d-2")) + on_memory = one.get_stream("d-2", payload_type=str, backend="memory") assert on_memory.ref.backend == "memory" - created = await one.create_stream[str]("d-2", backend="memory") + created = await one.create_stream("d-2", payload_type=str, backend="memory") assert created.ref.backend == "memory" @@ -430,30 +478,32 @@ async def test_stream_takes_the_named_backend_or_the_default(client: Client): MemoryStreamBackend(name="b"), ] two = Client(**config) - created = await two.create_stream[str]("moving", backend="b") - await created.publish("on b") - on_b = two.get_stream[str]("moving", backend="b") - assert on_b.ref == created.ref and await on_b.get_offset() > 0 - assert two.get_stream[str]("moving").ref.backend == "a" + created = await two.create_stream("moving", payload_type=str, backend="b") + await created.append("on b") + on_b = two.get_stream("moving", payload_type=str, backend="b") + assert on_b.ref == created.ref and await on_b.get_offset() > _BEGINNING + assert two.get_stream("moving", payload_type=str).ref.backend == "a" # A ref with no backend name uses the default backend, which doesn't have the Stream. with pytest.raises(streams.StreamNotFoundError): - await two.to_stream(streams.StreamRef(id="moving")).get_offset() + await two.get_stream(streams.StreamRef(id="moving")).get_offset() with pytest.raises(ValueError): - two.get_stream[str]("") + two.get_stream("", payload_type=str) @workflow.defn class WriteToStream: @workflow.run async def run(self, ref: streams.StreamRef[Turn]) -> None: - await workflow.to_stream(ref).publish(Turn("workflow")) + await workflow.get_stream(ref).append(Turn("workflow")) async def test_client_stream_takes_workflow_writes(stream_client: Client): - stream = await stream_client.create_stream[Turn](f"chat-{uuid.uuid4()}") - assert await stream.get_offset() == 0 - first = await stream.publish(Turn("client")) - last = await stream.publish(Turn("a"), Turn("b")) + stream = await stream_client.create_stream( + f"chat-{uuid.uuid4()}", payload_type=Turn + ) + assert await stream.get_offset() == _BEGINNING + first = await stream.append(Turn("client")) + last = await stream.append(Turn("a"), Turn("b")) assert last > first async with new_worker(stream_client, WriteToStream) as worker: await stream_client.execute_workflow( @@ -463,15 +513,20 @@ async def test_client_stream_takes_workflow_writes(stream_client: Client): task_queue=worker.task_queue, ) await stream.close() - items = [item async for item in stream.subscribe()] + items = [item async for item in stream.read()] data = [item.data for item in items if isinstance(item, streams.Record)] assert data == [Turn("client"), Turn("a"), Turn("b"), Turn("workflow")] - # Another Stream of the same id reads it from an offset. - again = stream_client.get_stream[Turn](stream.ref.id) + # Another Stream of the same id reads it from the offset after the first Record. + again = stream_client.get_stream(stream.ref.id, payload_type=Turn) + head = again.read(offset=first) + await anext(head) + after_first = head.next_offset + await head.aclose() + assert after_first is not None resumed = [ item.data - async for item in again.subscribe(offset=first + 1) + async for item in again.read(offset=after_first) if isinstance(item, streams.Record) ] assert resumed == [Turn("a"), Turn("b"), Turn("workflow")] @@ -485,10 +540,10 @@ def __init__(self) -> None: @workflow.run async def run(self, prompt: str) -> str: await workflow.sleep(timedelta(milliseconds=200)) - self._tokens = stream = await workflow.create_stream[str]( - f"{workflow.info().workflow_id}-tokens" + self._tokens = stream = await workflow.create_stream( + f"{workflow.info().workflow_id}-tokens", payload_type=str ) - await stream.publish(*prompt.split()) + await stream.append(*prompt.split()) await stream.close() return prompt.upper() @@ -507,8 +562,8 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): while ref is None: await asyncio.sleep(0.05) ref = await handle.query(Agent.stream) - reader = stream_client.to_stream(ref) - items = [item async for item in reader.subscribe()] + reader = stream_client.get_stream(ref) + items = [item async for item in reader.read()] await handle.result() assert [item.data for item in items if isinstance(item, streams.Record)] == [ "hi", @@ -521,7 +576,7 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): async def test_empty_stream_id_raises_with_no_call(client: Client): client, extension = fake_client(client) with pytest.raises(ValueError, match="id is empty"): - client.get_stream[Turn]("") + client.get_stream("", payload_type=Turn) assert not extension.namespaces @@ -532,8 +587,8 @@ async def test_registration_rules(client: Client): MemoryStreamBackend(name="b", default=True), ] two = Client(**config) - assert (await two.create_stream[str]("d-1")).ref.backend == "b" - assert two.get_stream[str]("d-1").ref.backend == "b" + assert (await two.create_stream("d-1", payload_type=str)).ref.backend == "b" + assert two.get_stream("d-1", payload_type=str).ref.backend == "b" # A Worker serves only one backend, and so does a Replayer. with pytest.raises(streams.StreamConfigError, match="one backend"): new_worker(Client(**config), Agent) diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 6a947a63b1..718797e4c7 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -41,6 +41,7 @@ OPEN = models.StreamStatus.StreamStatusOpen CLOSED = models.StreamStatus.StreamStatusClosed +USE_EXISTING = streams.StreamIDConflictPolicy.USE_EXISTING @dataclass @@ -64,10 +65,10 @@ async def read( payload_type: type[Any] = str, backend: str | None = None, ) -> list[object]: - stream = handle._client.get_stream[payload_type]( - stream_id(handle, name), backend=backend + stream = handle._client.get_stream( + stream_id(handle, name), payload_type=payload_type, backend=backend ) - return [item async for item in stream.subscribe(wait_new_records=False)] + return [item async for item in stream.read(wait_new_records=False)] async def status(handle: WorkflowHandle[Any, Any], name: str = "tokens") -> object: @@ -87,17 +88,17 @@ class Agent: @workflow.run async def run(self, prompt: str) -> str: # Two channels are two Streams. - turns = await workflow.create_stream[Turn](own_id("turns")) - tokens = await workflow.create_stream[str](own_id("tokens")) - await turns.publish(Turn("user", prompt), Turn("model")) - await tokens.publish(*prompt.split()) + turns = await workflow.create_stream(own_id("turns"), payload_type=Turn) + tokens = await workflow.create_stream(own_id("tokens"), payload_type=str) + await turns.append(Turn("user", prompt), Turn("model")) + await tokens.append(*prompt.split()) # A run's end doesn't close these Streams, so the run closes them before it returns. await turns.close() await tokens.close() return prompt.upper() -async def test_create_publish_and_close(stream_client: Client): +async def test_create_append_and_close(stream_client: Client): async with new_worker(stream_client, Agent) as worker: handle = await stream_client.start_workflow( Agent.run, "hi there", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue @@ -114,25 +115,27 @@ async def test_create_publish_and_close(stream_client: Client): class CreatesTwice: @workflow.run async def run(self, client_id: str) -> list[str]: - first = await workflow.create_stream[str](own_id()) + first = await workflow.create_stream(own_id(), payload_type=str) # The second create is the same create as the first, so it gets the same Stream. - again = await workflow.create_stream[str](own_id()) + again = await workflow.create_stream(own_id(), payload_type=str) results = [str(again.ref == first.ref)] await first.close() try: - await workflow.create_stream[str](client_id) + await workflow.create_stream(client_id, payload_type=str) except streams.StreamAlreadyExistsError as err: results.append(reason(err)) - # The create-or-get gets the Stream that exists, and ignores its options. - theirs = await workflow.create_or_get_stream[str]( - client_id, retention=timedelta(hours=1) + # USE_EXISTING gets the Stream that exists. + theirs = await workflow.create_stream( + client_id, payload_type=str, id_conflict_policy=USE_EXISTING ) - await theirs.publish("one") + await theirs.append("one") return results async def test_create_fails_on_an_id_that_another_caller_made(stream_client: Client): - theirs = await stream_client.create_stream[str](f"client-{uuid.uuid4()}") + theirs = await stream_client.create_stream( + f"client-{uuid.uuid4()}", payload_type=str + ) async with new_worker(stream_client, CreatesTwice) as worker: result = await stream_client.execute_workflow( CreatesTwice.run, @@ -141,7 +144,7 @@ async def test_create_fails_on_an_id_that_another_caller_made(stream_client: Cli task_queue=worker.task_queue, ) assert result == ["True", "STREAM_ERROR_REASON_ALREADY_EXISTS"] - items = [item async for item in theirs.subscribe(wait_new_records=False)] + items = [item async for item in theirs.read(wait_new_records=False)] assert data(items) == ["one"] @@ -150,8 +153,10 @@ class CreatesBetweenDraws: @workflow.run async def run(self) -> bool: before = workflow.random().getstate() - stream = await workflow.create_stream[str](own_id()) - workflow.create_or_get_stream_nowait[str](own_id()) + stream = await workflow.create_stream(own_id(), payload_type=str) + workflow.create_stream_nowait( + own_id(), payload_type=str, id_conflict_policy=USE_EXISTING + ) drawn = workflow.random().getstate() != before await stream.close() return drawn @@ -171,16 +176,19 @@ async def test_create_leaves_the_workflow_random_values_alone(stream_client: Cli class GeneratedCalls: @workflow.run async def run(self) -> list[str]: - tokens = await workflow.create_stream[str](own_id()) - generated = generated_stream[str]( - id=tokens.ref.id, owner=tokens.ref.owner, backend=tokens.ref.backend or None + tokens = await workflow.create_stream(own_id(), payload_type=str) + generated = generated_stream( + id=tokens.ref.id, + owner=tokens.ref.owner, + backend=tokens.ref.backend or None, + body_type=str, ) offset = await generated.append("a", "b") refused = [] try: await generated.read() - except streams.StreamUnsupportedError as err: - refused.append(type(err).__name__) + except streams.StreamError as err: + refused.append(reason(err)) await tokens.close() return [str(offset > 0), *refused] @@ -190,7 +198,7 @@ async def test_generated_workflow_calls_run_as_local_activities(stream_client: C handle = await stream_client.start_workflow( GeneratedCalls.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) - assert await handle.result() == ["True", "StreamUnsupportedError"] + assert await handle.result() == ["True", "STREAM_ERROR_REASON_UNSUPPORTED"] tokens = await read(handle) events = await _event_types(handle) assert data(tokens) == ["a", "b"] @@ -203,7 +211,7 @@ async def test_generated_workflow_calls_run_as_local_activities(stream_client: C class RefusedCalls: @workflow.run async def run(self) -> list[str]: - tokens = await workflow.create_stream[str](own_id()) + tokens = await workflow.create_stream(own_id(), payload_type=str) refused = [] # A Stream bound to another run chain fails before the block runs, and before any call. owner = streams.Execution( @@ -211,16 +219,16 @@ async def run(self) -> list[str]: business_id="other", run_id="run-1", ) - other = workflow.to_stream(streams.StreamRef[str](id="other", owner=owner)) + other = workflow.get_stream(streams.StreamRef[str](id="other", owner=owner)) try: async with other: refused.append("entered") - except streams.StreamNotOwnerError as err: - refused.append(type(err).__name__) + except streams.StreamError as err: + refused.append(reason(err)) try: await other.close() - except streams.StreamNotOwnerError as err: - refused.append(type(err).__name__) + except streams.StreamError as err: + refused.append(reason(err)) await tokens.close() return refused @@ -230,18 +238,25 @@ async def test_calls_that_workflow_code_cant_make_raise(stream_client: Client): result = await stream_client.execute_workflow( RefusedCalls.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) - assert result == ["StreamNotOwnerError", "StreamNotOwnerError"] + assert result == [ + "STREAM_ERROR_REASON_NOT_OWNER", + "STREAM_ERROR_REASON_NOT_OWNER", + ] @workflow.defn class Blocks: @workflow.run async def run(self) -> None: - async with await workflow.create_stream[str](own_id("first")) as first: - await first.publish("token-0") + async with await workflow.create_stream( + own_id("first"), payload_type=str + ) as first: + await first.append("token-0") try: - async with await workflow.create_stream[str](own_id("second")) as second: - await second.publish("token-1") + async with await workflow.create_stream( + own_id("second"), payload_type=str + ) as second: + await second.append("token-1") raise ApplicationError("turn failed") except ApplicationError: pass @@ -263,12 +278,12 @@ class Continuing: @workflow.run async def run(self, ref: streams.StreamRef[str] | None) -> str: if ref is None: - created = await workflow.create_stream[str](own_id()) - await created.publish("run 1") + created = await workflow.create_stream(own_id(), payload_type=str) + await created.append("run 1") # The next run gets the Stream only through the ref that this run forwards. workflow.continue_as_new(created.ref) - stream = workflow.to_stream(ref) - await stream.publish("run 2") + stream = workflow.get_stream(ref) + await stream.append("run 2") await stream.close() return "done" @@ -288,11 +303,11 @@ class CreatesInEachRun: @workflow.run async def run(self, first: bool) -> str: # The next run sends the create of the first run, so it gets the same Stream. - stream = await workflow.create_stream[str](own_id()) + stream = await workflow.create_stream(own_id(), payload_type=str) if first: - await stream.publish("run 1") + await stream.append("run 1") workflow.continue_as_new(False) - await stream.publish("run 2") + await stream.append("run 2") await stream.close() return "done" @@ -317,27 +332,29 @@ def __init__(self) -> None: @workflow.run async def run(self) -> list[str]: errors = [] - stream = await workflow.create_stream[str](own_id()) + stream = await workflow.create_stream(own_id(), payload_type=str) self._stream = stream - await stream.publish("last") + await stream.append("last") await stream.close() try: - await stream.publish("late") + await stream.append("late") except streams.StreamClosedError as err: errors.append(reason(err)) # A closed Stream fails a create with CLOSED, which comes before ALREADY_EXISTS. - for create in (workflow.create_stream, workflow.create_or_get_stream): + for policy in (streams.StreamIDConflictPolicy.FAIL, USE_EXISTING): try: - await create[str](own_id()) + await workflow.create_stream( + own_id(), payload_type=str, id_conflict_policy=policy + ) except streams.StreamClosedError as err: errors.append(reason(err)) return errors @workflow.query - def publish_in_query(self) -> str: + def append_in_query(self) -> str: assert self._stream is not None try: - self._stream.publish("x") + self._stream.append("x") except workflow.ReadOnlyContextError as err: return type(err).__name__ return "wrote" @@ -351,7 +368,7 @@ async def test_replay_reproducible_errors_raise_in_workflow_code( Mistakes.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await handle.result() == ["STREAM_ERROR_REASON_CLOSED"] * 3 - assert await handle.query(Mistakes.publish_in_query) == "ReadOnlyContextError" + assert await handle.query(Mistakes.append_in_query) == "ReadOnlyContextError" assert data(await read(handle)) == ["last"] @@ -359,8 +376,8 @@ async def test_replay_reproducible_errors_raise_in_workflow_code( class ClosesAReceivedStream: @workflow.run async def run(self, ref: streams.StreamRef[str]) -> str: - stream = workflow.to_stream(ref) - await stream.publish("from workflow") + stream = workflow.get_stream(ref) + await stream.append("from workflow") await stream.close() return "closed" @@ -369,7 +386,9 @@ async def test_workflow_closes_a_standalone_stream_that_it_received( stream_client: Client, ): # A standalone Stream has no owner, so a Workflow closes it as its run's Producer. - stream = await stream_client.create_stream[str](f"received-{uuid.uuid4()}") + stream = await stream_client.create_stream( + f"received-{uuid.uuid4()}", payload_type=str + ) async with new_worker(stream_client, ClosesAReceivedStream) as worker: assert ( await stream_client.execute_workflow( @@ -380,7 +399,7 @@ async def test_workflow_closes_a_standalone_stream_that_it_received( ) == "closed" ) - items = [item async for item in stream.subscribe()] + items = [item async for item in stream.read()] assert data(items) == ["from workflow"] assert await stream_status(stream_client, stream.ref.id) == CLOSED @@ -389,8 +408,8 @@ async def test_workflow_closes_a_standalone_stream_that_it_received( class Failing: @workflow.run async def run(self) -> None: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("before") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("before") raise ApplicationError("run failed", non_retryable=True) @@ -411,8 +430,8 @@ class RetriesItsCreate: @workflow.run async def run(self) -> str: attempt = workflow.info().attempt - stream = await workflow.create_stream[str](own_id()) - await stream.publish(f"run-{attempt}") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append(f"run-{attempt}") if attempt == 1: raise ApplicationError("the first run fails") await stream.close() @@ -440,8 +459,8 @@ class FailsInItsBlockOnce: @workflow.run async def run(self) -> str: attempt = workflow.info().attempt - async with await workflow.create_stream[str](own_id()) as stream: - await stream.publish(f"run-{attempt}") + async with await workflow.create_stream(own_id(), payload_type=str) as stream: + await stream.append(f"run-{attempt}") if attempt == 1: raise ApplicationError("the first run fails") return f"done on attempt {attempt}" @@ -473,8 +492,10 @@ class EndsItsBlock: @workflow.run async def run(self, mode: str) -> str: try: - async with await workflow.create_stream[str](own_id()) as stream: - await stream.publish(mode) + async with await workflow.create_stream( + own_id(), payload_type=str + ) as stream: + await stream.append(mode) if mode == "non_retryable": raise ApplicationError("block failed", non_retryable=True) if mode == "listed_type": @@ -550,8 +571,8 @@ async def waits_for_cancel() -> None: class CancelledInItsBlock: @workflow.run async def run(self, local: bool) -> None: - async with await workflow.create_stream[str](own_id()) as stream: - await stream.publish("started") + async with await workflow.create_stream(own_id(), payload_type=str) as stream: + await stream.append("started") if local: await workflow.execute_local_activity( waits_for_cancel, start_to_close_timeout=timedelta(minutes=1) @@ -592,8 +613,8 @@ class CronBlock: @workflow.run async def run(self) -> str: # Each cron run sends the create of the first run, so it gets the same Stream. - async with await workflow.create_stream[str](own_id()) as stream: - await stream.publish("turn") + async with await workflow.create_stream(own_id(), payload_type=str) as stream: + await stream.append("turn") return "done" @@ -606,7 +627,7 @@ async def test_block_end_leaves_a_cron_stream_open(stream_client: Client): task_queue=worker.task_queue, cron_schedule="@every 1s", ) - reader = stream_client.get_stream[str](stream_id(handle)).subscribe( + reader = stream_client.get_stream(stream_id(handle), payload_type=str).read( create_wait=timedelta(seconds=20) ) first = await asyncio.wait_for(anext(reader), 20) @@ -624,8 +645,8 @@ async def test_block_end_leaves_a_cron_stream_open(stream_client: Client): class SchedulesACloseThenFailsItsTask: @workflow.run async def run(self) -> None: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("token") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("token") stream.close_nowait() workflow_id = workflow.info().workflow_id if workflow_id not in failed_close_tasks: @@ -674,7 +695,7 @@ async def test_failed_task_writes_no_close(stream_client: Client): class CreatesOn: @workflow.run async def run(self, backend: str | None) -> None: - await workflow.create_stream[str](own_id(), backend=backend) + await workflow.create_stream(own_id(), payload_type=str, backend=backend) async def test_worker_config_errors_fail_the_task( @@ -702,7 +723,7 @@ class SyncHandlers: @workflow.init def __init__(self, prompt: str) -> None: # The create runs first, so the Stream exists for the signal's writes. - self._stream = workflow.create_stream_nowait[str](own_id()) + self._stream = workflow.create_stream_nowait(own_id(), payload_type=str) self._prompt = prompt self._done = False @@ -713,13 +734,13 @@ async def run(self, prompt: str) -> None: @workflow.signal def token(self, text: str) -> None: - self._stream.publish_nowait(text, self._prompt) + self._stream.append_nowait(text, self._prompt) @workflow.update def finish(self) -> str: self._stream.close_nowait() try: - self._stream.publish_nowait("late") + self._stream.append_nowait("late") except streams.StreamClosedError as err: return reason(err) finally: @@ -727,7 +748,7 @@ def finish(self) -> str: return "wrote" -async def test_sync_handlers_publish_and_close_with_no_await(stream_client: Client): +async def test_sync_handlers_append_and_close_with_no_await(stream_client: Client): async with new_worker(stream_client, SyncHandlers) as worker: handle = await stream_client.start_workflow( SyncHandlers.run, "p", id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue @@ -746,17 +767,19 @@ async def test_sync_handlers_publish_and_close_with_no_await(stream_client: Clie class CreatesInInit: @workflow.init def __init__(self, prompt: str) -> None: - self._stream = workflow.create_or_get_stream_nowait[str](own_id()) - self._stream.publish_nowait(prompt) + self._stream = workflow.create_stream_nowait( + own_id(), payload_type=str, id_conflict_policy=USE_EXISTING + ) + self._stream.append_nowait(prompt) @workflow.run async def run(self, prompt: str) -> str: - await self._stream.publish(prompt.upper()) + await self._stream.append(prompt.upper()) await self._stream.close() return "done" -async def test_init_creates_and_publishes_with_no_await(stream_client: Client): +async def test_init_creates_and_appends_with_no_await(stream_client: Client): async with new_worker(stream_client, CreatesInInit) as worker: handle = await stream_client.start_workflow( CreatesInInit.run, @@ -774,8 +797,8 @@ class CreatesTwoInInit: @workflow.init def __init__(self, taken_id: str) -> None: # The first create fails while the Local Activity of the second one is in flight. - workflow.create_stream_nowait[str](taken_id) - workflow.create_stream_nowait[str](own_id()) + workflow.create_stream_nowait(taken_id, payload_type=str) + workflow.create_stream_nowait(own_id(), payload_type=str) @workflow.run async def run(self, _taken_id: str) -> str: @@ -785,7 +808,7 @@ async def run(self, _taken_id: str) -> str: async def test_nowait_error_raises_after_the_other_writes(stream_client: Client): # A command after the run's fail command fails the Workflow Task, and each retry fails # the same way, so the run would never end. - taken = await stream_client.create_stream[str](f"taken-{uuid.uuid4()}") + taken = await stream_client.create_stream(f"taken-{uuid.uuid4()}", payload_type=str) async with new_worker(stream_client, CreatesTwoInInit) as worker: handle = await stream_client.start_workflow( CreatesTwoInInit.run, @@ -809,11 +832,11 @@ async def test_nowait_error_raises_after_the_other_writes(stream_client: Client) class WritesAfterAFailedCreate: @workflow.run async def run(self, taken_id: str, wait: bool) -> None: - stream = workflow.create_stream_nowait[str](taken_id) + stream = workflow.create_stream_nowait(taken_id, payload_type=str) if wait: - await stream.publish("from the run") + await stream.append("from the run") else: - stream.publish_nowait("from the run") + stream.append_nowait("from the run") @pytest.mark.parametrize("wait", [False, True]) @@ -822,7 +845,7 @@ async def test_writes_behind_a_failed_create_send_nothing( ): # The Stream of a failed create can be another owner's, so the writes queued behind the # create must not reach it, and the run fails. - taken = await stream_client.create_stream[str](f"taken-{uuid.uuid4()}") + taken = await stream_client.create_stream(f"taken-{uuid.uuid4()}", payload_type=str) async with new_worker(stream_client, WritesAfterAFailedCreate) as worker: handle = await stream_client.start_workflow( WritesAfterAFailedCreate.run, @@ -834,26 +857,26 @@ async def test_writes_behind_a_failed_create_send_nothing( await asyncio.wait_for(handle.result(), 20) assert isinstance(failed.value.cause, ApplicationError) assert failed.value.cause.type == "StreamAlreadyExistsError" - assert [item async for item in taken.subscribe(wait_new_records=False)] == [] + assert [item async for item in taken.read(wait_new_records=False)] == [] @workflow.defn -class PublishesWithNoAwait: +class AppendsWithNoAwait: @workflow.run - async def run(self) -> list[int]: - tokens = await workflow.create_stream[str](own_id()) - first = tokens.publish("a") - second = tokens.publish("b") - last = await tokens.publish("c") + async def run(self) -> list[str]: + tokens = await workflow.create_stream(own_id(), payload_type=str) + first = tokens.append("a") + second = tokens.append("b") + last = await tokens.append("c") # Nobody awaits this write, and the run's end still waits for it. - tokens.publish("d") - return [first.result(), second.result(), last] + tokens.append("d") + return [str(first.result()), str(second.result()), str(last)] -async def test_publish_starts_the_write_and_returns_its_future(stream_client: Client): - async with new_worker(stream_client, PublishesWithNoAwait) as worker: +async def test_append_starts_the_write_and_returns_its_future(stream_client: Client): + async with new_worker(stream_client, AppendsWithNoAwait) as worker: handle = await stream_client.start_workflow( - PublishesWithNoAwait.run, + AppendsWithNoAwait.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -861,16 +884,16 @@ async def test_publish_starts_the_write_and_returns_its_future(stream_client: Cl items = await read(handle) written = [item for item in items if isinstance(item, streams.Record)] assert [r.data for r in written] == ["a", "b", "c", "d"] - assert offsets == [r.offset for r in written[:3]] + assert offsets == [str(r.offset) for r in written[:3]] @workflow.defn -class LeavesAFailedPublish: +class LeavesAFailedAppend: @workflow.run async def run(self, stream_id: str, retrieve: bool) -> str: # Nobody created the Stream, so the backend refuses the write. - missing = workflow.to_stream(streams.StreamRef[str](id=stream_id)) - future = missing.publish("lost") + missing = workflow.get_stream(streams.StreamRef[str](id=stream_id)) + future = missing.append("lost") errors: list[BaseException | None] = [] if retrieve: future.add_done_callback(lambda done: errors.append(done.exception())) @@ -878,15 +901,15 @@ async def run(self, stream_id: str, retrieve: bool) -> str: return "done" -async def test_publish_error_that_no_caller_gets_raises_at_the_run_end( +async def test_append_error_that_no_caller_gets_raises_at_the_run_end( stream_client: Client, caplog: pytest.LogCaptureFixture ): - # The error raises as if the run didn't catch it, as with publish_nowait. + # The error raises as if the run didn't catch it, as with append_nowait. lost_id = f"missing-{uuid.uuid4()}" with caplog.at_level(logging.WARNING, logger="temporalio.workflow"): - async with new_worker(stream_client, LeavesAFailedPublish) as worker: + async with new_worker(stream_client, LeavesAFailedAppend) as worker: lost = await stream_client.start_workflow( - LeavesAFailedPublish.run, + LeavesAFailedAppend.run, args=[lost_id, False], id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, @@ -897,7 +920,7 @@ async def test_publish_error_that_no_caller_gets_raises_at_the_run_end( assert failed.value.cause.type == "StreamNotFoundError" # A done callback that reads the error owns it, so the run completes. owned = await stream_client.start_workflow( - LeavesAFailedPublish.run, + LeavesAFailedAppend.run, args=[f"missing-{uuid.uuid4()}", True], id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, @@ -913,17 +936,16 @@ async def test_publish_error_that_no_caller_gets_raises_at_the_run_end( @workflow.defn class Gathers: @workflow.run - async def run(self) -> list[int]: - tokens = await workflow.create_stream[str](own_id()) - tokens.publish_nowait("0") + async def run(self) -> list[str]: + tokens = await workflow.create_stream(own_id(), payload_type=str) + tokens.append_nowait("0") # The first write starts, so the next ones wait behind it. await asyncio.sleep(0) - return list( - await asyncio.gather(*(tokens.publish(str(n)) for n in range(1, 5))) - ) + results = await asyncio.gather(*(tokens.append(str(n)) for n in range(1, 5))) + return [str(offset) for offset in results] -async def test_concurrent_publishes_keep_their_order(stream_client: Client): +async def test_concurrent_appends_keep_their_order(stream_client: Client): async with new_worker(stream_client, Gathers) as worker: handle = await stream_client.start_workflow( Gathers.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue @@ -935,7 +957,7 @@ async def test_concurrent_publishes_keep_their_order(stream_client: Client): # of its own Record. assert ( offsets - == [item.offset for item in items if isinstance(item, streams.Record)][1:] + == [str(item.offset) for item in items if isinstance(item, streams.Record)][1:] ) history = await handle.fetch_history() markers = [ @@ -1042,18 +1064,18 @@ async def loses_the_first_answer( @workflow.defn -class PublishesToAMissingStream: +class AppendsToAMissingStream: @workflow.run async def run(self) -> None: # No create, so the Store refuses the append, and no retry sends it again. - stream = workflow.to_stream(streams.StreamRef[str](id=own_id())) - await stream.publish("x" * 100_000) + stream = workflow.get_stream(streams.StreamRef[str](id=own_id())) + await stream.append("x" * 100_000) async def test_refused_write_keeps_no_pending_bytes(stream_client: Client): - async with new_worker(stream_client, PublishesToAMissingStream) as worker: + async with new_worker(stream_client, AppendsToAMissingStream) as worker: handle = await stream_client.start_workflow( - PublishesToAMissingStream.run, + AppendsToAMissingStream.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, ) @@ -1087,24 +1109,23 @@ def test_pending_appends_keep_the_newest_epoch_and_a_bounded_size(): @workflow.defn -class PublishesLargeValues: +class AppendsLargeValues: @workflow.run - async def run(self, size: int) -> list[int]: - tokens = await workflow.create_stream[str](own_id()) - return list( - await asyncio.gather( - tokens.publish(*["a" * size] * 3), tokens.publish(*["b" * size] * 3) - ) + async def run(self, size: int) -> list[str]: + tokens = await workflow.create_stream(own_id(), payload_type=str) + results = await asyncio.gather( + tokens.append(*["a" * size] * 3), tokens.append(*["b" * size] * 3) ) + return [str(offset) for offset in results] async def test_merge_that_the_codec_grows_past_the_batch_limit(stream_client: Client): - # The two publishes merge under the Workflow's cap. The codec makes the merge larger than the - # batch limit, so each publish is sent as its own batch. + # The two appends merge under the Workflow's cap. The codec makes the merge larger than the + # batch limit, so each append is sent as its own batch. client = with_codec(stream_client, _GrowingCodec()) - async with new_worker(client, PublishesLargeValues) as worker: + async with new_worker(client, AppendsLargeValues) as worker: handle = await client.start_workflow( - PublishesLargeValues.run, + AppendsLargeValues.run, 300_000, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, @@ -1113,26 +1134,26 @@ async def test_merge_that_the_codec_grows_past_the_batch_limit(stream_client: Cl items = await read(handle) written = [item for item in items if isinstance(item, streams.Record)] assert [len(r.data) for r in written] == [300_000] * 6 - assert offsets == [written[2].offset, written[5].offset] + assert offsets == [str(written[2].offset), str(written[5].offset)] @workflow.defn -class PublishesTooMuch: +class AppendsTooMuch: @workflow.run async def run(self) -> str: - tokens = await workflow.create_stream[str](own_id()) + tokens = await workflow.create_stream(own_id(), payload_type=str) try: - await tokens.publish(*["x" * 1_400_000] * 4) - except streams.StreamTooLargeError as err: + await tokens.append(*["x" * 1_400_000] * 4) + except streams.StreamError as err: return reason(err) return "written" async def test_batch_over_the_store_limit_raises_too_large(stream_client: Client): - # Each Record is under the Store's Record limit, and the publish's batch is over 4 MiB. - async with new_worker(stream_client, PublishesTooMuch) as worker: + # Each Record is under the Store's Record limit, and the append's batch is over 4 MiB. + async with new_worker(stream_client, AppendsTooMuch) as worker: handle = await stream_client.start_workflow( - PublishesTooMuch.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + AppendsTooMuch.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) assert await handle.result() == "STREAM_ERROR_REASON_TOO_LARGE" assert data(await read(handle)) == [] @@ -1142,16 +1163,18 @@ async def test_batch_over_the_store_limit_raises_too_large(stream_client: Client class CreatesOnTheDefault: @workflow.run async def run(self) -> None: - await workflow.create_stream[str](own_id()) + await workflow.create_stream(own_id(), payload_type=str) @workflow.defn class CreatesOnANamedBackend: @workflow.run async def run(self, backend: str) -> str: - stream = await workflow.create_stream[str](own_id(), backend=backend) - await stream.publish(f"on {backend}") - await workflow.to_stream(stream.ref).close() + stream = await workflow.create_stream( + own_id(), payload_type=str, backend=backend + ) + await stream.append(f"on {backend}") + await workflow.get_stream(stream.ref).close() return stream.ref.backend or "" @@ -1159,22 +1182,30 @@ async def run(self, backend: str) -> str: class CreatesAfterAnIdleTime: @workflow.run async def run(self) -> str: - await workflow.create_stream[str]( - own_id(), retention=timedelta(milliseconds=500) - ) + await workflow.create_stream(own_id(), payload_type=str) await workflow.sleep(timedelta(seconds=1.5)) try: - await workflow.create_or_get_stream[str](own_id()) - except streams.StreamExpiredError as err: - return reason(err) + await workflow.create_stream( + own_id(), payload_type=str, id_conflict_policy=USE_EXISTING + ) + except streams.StreamError as err: + if err.reason == StreamErrorReason.STREAM_ERROR_REASON_EXPIRED: + return reason(err) + raise return "created" -async def test_create_of_an_expired_workflow_stream_raises(stream_client: Client): - # A Workflow's standalone Stream expires one Retention after its last write. - async with new_worker(stream_client, CreatesAfterAnIdleTime) as worker: +async def test_create_of_an_expired_workflow_stream_raises(client: Client): + # Retention is a backend setting now. A Workflow's standalone Stream expires one Retention + # after its last write. + config = client.config() + config["stream_backends"] = [ + MemoryStreamBackend(retention=timedelta(milliseconds=500), default=True) + ] + memory_client = Client(**config) + async with new_worker(memory_client, CreatesAfterAnIdleTime) as worker: assert ( - await stream_client.execute_workflow( + await memory_client.execute_workflow( CreatesAfterAnIdleTime.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue, @@ -1231,8 +1262,8 @@ async def _cancel_is_handled(handle: WorkflowHandle[Any, Any]) -> None: class ClosesThenReturns: @workflow.run async def run(self) -> str: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("a") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("a") stream.close_nowait() return "done" @@ -1276,15 +1307,15 @@ def __init__(self) -> None: @workflow.run async def run(self) -> str: - self.late = await workflow.create_stream[str](own_id("late")) - tokens = await workflow.create_stream[str](own_id()) + self.late = await workflow.create_stream(own_id("late"), payload_type=str) + tokens = await workflow.create_stream(own_id(), payload_type=str) tokens.close_nowait() return "done" @workflow.signal def write(self) -> None: assert self.late is not None - self.late.publish_nowait("slow") + self.late.append_nowait("slow") async def _signal_is_handled(handle: WorkflowHandle[Any, Any]) -> None: @@ -1338,11 +1369,11 @@ def to_payloads(self, values: Sequence[Any]) -> list[Payload]: @workflow.defn -class PublishesOneValue: +class AppendsOneValue: @workflow.run async def run(self) -> None: - stream = await workflow.create_stream[str](own_id()) - await stream.publish("plain") + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("plain") await stream.close() @@ -1351,9 +1382,9 @@ async def test_workflow_records_get_no_serialization_context(stream_client: Clie config = stream_client.config() config["data_converter"] = DataConverter(payload_converter_class=_TagsWithContext) tagging = Client(**config) - async with new_worker(tagging, PublishesOneValue) as worker: + async with new_worker(tagging, AppendsOneValue) as worker: handle = await tagging.start_workflow( - PublishesOneValue.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue + AppendsOneValue.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue ) await handle.result() assert data(await read(handle)) == ["plain"] @@ -1364,11 +1395,13 @@ class ContinuesInItsBlock: @workflow.run async def run(self, ref: streams.StreamRef[str] | None) -> str: if ref is None: - async with await workflow.create_stream[str](own_id()) as stream: - await stream.publish("run 1") + async with await workflow.create_stream( + own_id(), payload_type=str + ) as stream: + await stream.append("run 1") workflow.continue_as_new(stream.ref) try: - await workflow.to_stream(ref).publish("run 2") + await workflow.get_stream(ref).append("run 2") except streams.StreamClosedError as err: return reason(err) return "wrote" From 59deec6b2df36cfd0128e1dc2c7fdd1264544e8b Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sat, 7 Nov 2026 18:14:43 -0700 Subject: [PATCH 3/7] Updated the Stream examples and chat app to the thinned facade --- .../examples/activity_streams_tokens.py | 30 ++--- .../streams/examples/agent_publishes_turns.py | 39 +++--- .../streams/examples/chat_app/README.md | 53 ++++---- .../streams/examples/chat_app/activities.py | 8 +- .../contrib/streams/examples/chat_app/docs.py | 6 +- .../streams/examples/chat_app/models/fake.py | 2 +- .../streams/examples/chat_app/server.py | 81 +++++++----- .../streams/examples/chat_app/shared.py | 3 +- .../examples/chat_app/static/index.html | 4 +- .../examples/chat_app/test_chat_app.py | 115 +++++++++++++----- .../streams/examples/chat_app/tools.py | 6 +- .../streams/examples/chat_app/workflow.py | 59 ++++----- .../streams/examples/client_creates_stream.py | 17 +-- .../streams/examples/resume_from_offset.py | 22 ++-- .../contrib/streams/examples/test_examples.py | 2 +- .../streams/examples/ui_backend_reads.py | 22 ++-- .../contrib/streams/examples/worker_setup.py | 16 +-- 17 files changed, 288 insertions(+), 197 deletions(-) diff --git a/tests/contrib/streams/examples/activity_streams_tokens.py b/tests/contrib/streams/examples/activity_streams_tokens.py index 0eac15b36d..33c415b651 100644 --- a/tests/contrib/streams/examples/activity_streams_tokens.py +++ b/tests/contrib/streams/examples/activity_streams_tokens.py @@ -2,7 +2,7 @@ The Workflow passes its Stream's ref to the Activity as a field of the input. An async Activity gets the Stream from ``activity.client()``, so it writes as the attempt, and batches the model's -deltas with ``BufferedPublisher``. A sync Activity has no ``activity.client()``, so it keeps the +deltas with ``BufferedAppender``. A sync Activity has no ``activity.client()``, so it keeps the Worker's client. It wraps its Stream in ``SyncStream``, whose calls block its thread, so it works in a thread pool executor. """ @@ -16,7 +16,7 @@ from datetime import timedelta from temporalio import activity, streams, workflow -from temporalio.client import BufferedPublisher, Client, SyncStream +from temporalio.client import BufferedAppender, Client, SyncStream from temporalio.worker import Worker Message = str @@ -65,11 +65,11 @@ class SummarizeInput: @activity.defn async def call_model(input: CallModelInput) -> Reply: - stream = activity.client().to_stream(input.tokens) # writes as this attempt + stream = activity.client().get_stream(input.tokens) # writes as this attempt response = llm.stream(input.messages) - async with BufferedPublisher(stream) as tokens: # batches deltas + async with BufferedAppender(stream) as tokens: # batches deltas async for delta in response: - tokens.publish(delta) + tokens.append(delta) return response.reply() @@ -78,10 +78,10 @@ def __init__(self, client: Client) -> None: self._client = client # the Worker's client @activity.defn - def summarize(self, input: SummarizeInput) -> int: - tokens = SyncStream(self._client.to_stream(input.tokens)) - # The sync publish blocks until the Record is written, and returns its offset. - return tokens.publish(f"summary: {input.text}") + def summarize(self, input: SummarizeInput) -> str: + tokens = SyncStream(self._client.get_stream(input.tokens)) + # The sync append blocks until the Record is written, and returns its offset. + return str(tokens.append(f"summary: {input.text}")) @workflow.defn @@ -94,9 +94,9 @@ def stream(self) -> streams.StreamRef[str] | None: return self._tokens.ref if self._tokens else None @workflow.run - async def run(self, prompt: str) -> int: - self._tokens = tokens = await workflow.create_stream[str]( - f"{workflow.info().workflow_id}-tokens" + async def run(self, prompt: str) -> str: + self._tokens = tokens = await workflow.create_stream( + f"{workflow.info().workflow_id}-tokens", payload_type=str ) reply = await workflow.execute_activity( call_model, @@ -114,7 +114,7 @@ async def run(self, prompt: str) -> int: async def main( client: Client, task_queue: str -) -> tuple[int, list[streams.Record[str]]]: +) -> tuple[str, list[streams.Record[str]]]: with concurrent.futures.ThreadPoolExecutor() as executor: async with Worker( client, @@ -132,6 +132,6 @@ async def main( summary = await handle.result() ref = await handle.query(Model.stream) assert ref is not None - stream = client.to_stream(ref) - records = [r async for r in stream.subscribe() if isinstance(r, streams.Record)] + stream = client.get_stream(ref) + records = [r async for r in stream.read() if isinstance(r, streams.Record)] return summary, records diff --git a/tests/contrib/streams/examples/agent_publishes_turns.py b/tests/contrib/streams/examples/agent_publishes_turns.py index 1b1a7fdec5..c55f0a0186 100644 --- a/tests/contrib/streams/examples/agent_publishes_turns.py +++ b/tests/contrib/streams/examples/agent_publishes_turns.py @@ -1,8 +1,8 @@ """Streams API proposal, "An agent Workflow publishes its turns". The Workflow creates its two Streams, turns and token deltas, under ids of its choice, and hands -out the turns Stream's ref through a ``stream`` query. It publishes each turn with an awaited -``publish``, and passes the tokens ref to the model Activity in its one input object. A run +out the turns Stream's ref through a ``stream`` query. It appends each turn with an awaited +``append``, and passes the tokens ref to the model Activity in its one input object. A run after continue-as-new gets the earlier run's refs and creates nothing. It closes both Streams before it returns, since a run's end doesn't close a Stream. """ @@ -14,7 +14,7 @@ from datetime import timedelta from temporalio import activity, streams, workflow -from temporalio.client import BufferedPublisher, Client +from temporalio.client import BufferedAppender, Client from temporalio.worker import Worker MINUTE = timedelta(minutes=1) @@ -42,10 +42,10 @@ class CallModelInput: @activity.defn async def call_model(input: CallModelInput) -> Reply: - stream = activity.client().to_stream(input.tokens) # writes as this attempt - async with BufferedPublisher(stream) as tokens: # batches deltas + stream = activity.client().get_stream(input.tokens) # writes as this attempt + async with BufferedAppender(stream) as tokens: # batches deltas for delta in input.messages[-1].split(): - tokens.publish(delta) + tokens.append(delta) if len(input.messages) == 1: return Reply("", tool_calls=["search"]) return Reply("the answer") @@ -80,29 +80,29 @@ async def run(self, input: AgentInput) -> str: # A Workflow creates only standalone Streams, under an id of its choice. A run after # Continue-as-New gets the earlier run's refs and creates nothing. if input.turns and input.tokens: - self._turns = turns = workflow.to_stream(input.turns) - tokens = workflow.to_stream(input.tokens) + self._turns = turns = workflow.get_stream(input.turns) + tokens = workflow.get_stream(input.tokens) else: - self._turns = turns = await workflow.create_stream[Turn]( - f"{input.session_id}-turns" - ) # the default Retention - # Token deltas matter only to live readers, so they get a short Retention. - tokens = await workflow.create_stream[str]( - f"{input.session_id}-tokens", retention=timedelta(hours=1) + # Retention is a backend setting now, so both Streams use the backend's Retention. + self._turns = turns = await workflow.create_stream( + f"{input.session_id}-turns", payload_type=Turn + ) + tokens = await workflow.create_stream( + f"{input.session_id}-tokens", payload_type=str ) while True: if workflow.info().is_continue_as_new_suggested(): workflow.continue_as_new( # forward the refs, so the next run keeps the Streams AgentInput(input.session_id, messages, turns.ref, tokens.ref) ) - await turns.publish(Turn("model")) # one Local Activity + await turns.append(Turn("model")) # one Local Activity reply = await workflow.execute_activity( call_model, CallModelInput(messages, tokens.ref), start_to_close_timeout=MINUTE, ) if not reply.tool_calls: - await turns.publish(Turn("done", reply.text)) + await turns.append(Turn("done", reply.text)) await turns.close() # the done Turn carries the reply await tokens.close() return reply.text @@ -132,5 +132,8 @@ async def main(client: Client, task_queue: str) -> list[streams.Record[Turn]]: # The Workflow hands out its turns ref, and the query's client reads the Stream. ref = await handle.query(Agent.stream) assert ref is not None - stream = client.to_stream(ref, payload_type=Turn) - return [r async for r in stream.subscribe() if isinstance(r, streams.Record)] + # A deserialized ref decodes as JSON, so the id form takes the payload type. + stream = client.get_stream( + ref.id, payload_type=Turn, owner=ref.owner, backend=ref.backend + ) + return [r async for r in stream.read() if isinstance(r, streams.Record)] diff --git a/tests/contrib/streams/examples/chat_app/README.md b/tests/contrib/streams/examples/chat_app/README.md index 3fc9615bcc..23868e887b 100644 --- a/tests/contrib/streams/examples/chat_app/README.md +++ b/tests/contrib/streams/examples/chat_app/README.md @@ -82,7 +82,7 @@ another in one chat, so a new viewer sees each case. - **Failed attempt.** The first attempt of the round fails after a few tokens. One second later, the retry's `Superseded` item crosses them out, and the round streams again. - **Message during a run.** While `search_docs` runs, the page sends a second message. The - `add_message` Signal publishes its Turn at once with `publish_nowait`, and the message joins + `add_message` Signal appends its Turn at once with `append_nowait`, and the message joins the next round of the same turn. While a turn runs, the composer sends this way too. - **Sub-agent.** `research` starts a child Workflow. It creates its own Streams, under ids with its run id that the parent can't know, and writes a Turn with their refs to the parent's turns @@ -113,43 +113,47 @@ another in one chat, so a new viewer sees each case. ## Where each feature is -- **`create_or_get_stream` in a Workflow.** [`ChatSession.__init__`](workflow.py) gets the turns - and the tokens Stream with `workflow.create_or_get_stream_nowait[...]`. A chat id can come back, - such as in a restarted session, and the run then gets the chat's Streams. -- **`create_or_get_stream` in a client.** The server's [`lifespan`](server.py) gets the lobby - Stream, which lists the chats. Each server start runs it. +- **`create_stream` with `USE_EXISTING` in a Workflow.** [`ChatSession.__init__`](workflow.py) + gets the turns and the tokens Stream with `workflow.create_stream_nowait(..., + id_conflict_policy=StreamIDConflictPolicy.USE_EXISTING)`. A chat id can come back, such as in a + restarted session, and the run then gets the chat's Streams. +- **`create_stream` with `USE_EXISTING` in a client.** The server's [`lifespan`](server.py) gets + the lobby Stream, which lists the chats. Each server start runs it. - **Strict create.** [`share`](server.py) creates each snapshot with - `client.create_stream[SharedTurn]`, since a share id names one snapshot. A second share of the - same point raises `StreamClosedError`, since that snapshot is closed. A share that runs at the - same time raises `StreamAlreadyExistsError`. Both give the same link. + `client.create_stream(payload_type=SharedTurn)`, since a share id names one snapshot. A second + share of the same point raises `StreamClosedError`, since that snapshot is closed. A share that + runs at the same time raises `StreamAlreadyExistsError`. Both give the same link. - **Standalone Streams that a Workflow creates.** `-turns` and `-tokens`, with ids that the Workflow picks. - **Refs as data.** The `streams` query gives `SessionStreams`. `ModelInput.tokens` takes a ref to the Activity, and `SessionInput.streams` takes both refs through continue-as-new. -- **`to_stream`.** `workflow.to_stream` in the next run, `activity.client().to_stream` in the - Activity, and `client.to_stream(ref, payload_type=Turn)` in the server. -- **`client.get_stream[T]` by id.** The share links, which name a snapshot's Stream id. -- **Awaited `publish`.** The `send` Update returns the user Turn's offset, which the page shows - under the composer. The server's lobby publish is awaited too. -- **`publish_nowait`.** `@workflow.init` publishes "Run N of the session started", and the - `add_message` Signal publishes the user's Turn, since neither can await. +- **`get_stream`.** `workflow.get_stream(ref)` in the next run, + `activity.client().get_stream(ref)` in the Activity, and `client.get_stream(id, + payload_type=Turn)` in the server. +- **`client.get_stream` by id.** The share links, which name a snapshot's Stream id. +- **Awaited `append`.** The `send` Update returns the user Turn's offset, which the page shows + under the composer. The server's lobby append is awaited too. +- **`append_nowait`.** `@workflow.init` appends "Run N of the session started", and the + `add_message` Signal appends the user's Turn, since neither can await. - **An agent loop.** [`Agent.answer`](workflow.py) runs the rounds of an answer, for a chat and for a sub-agent. Each round is the model Activity, and the round's tools run at the same time. - **Activities that write Turns.** [`run_tool`](tools.py) writes its progress and its result to the turns Stream, from a ref in its input. Each attempt of each tool is its own Producer. - **A child Workflow's own Streams.** [`Researcher`](workflow.py) creates its Streams at run time, - writes to its parent's turns Stream through `workflow.to_stream`, and puts its own refs in a + writes to its parent's turns Stream through `workflow.get_stream`, and puts its own refs in a Turn. A ref is data, also inside a Record. -- **`BufferedPublisher` and `flush`.** [`call_model`](activities.py) batches the deltas, and +- **`BufferedAppender` and `flush`.** [`call_model`](activities.py) batches the deltas, and flushes before the toggle fails the attempt. -- **`subscribe(offset=...)`.** [`session_events`](server.py) resumes each read at the offset that +- **`read(offset=...)`.** [`session_events`](server.py) resumes each read at the offset that the tab saved. - **`streams.END`.** A tab that joins reads only new tokens, since the turns Stream has the earlier replies. -- **`subscribe(last=...)`.** The lobby sidebar reads the newest 20 chats, then each new one. +- **`read(last=...)`.** The lobby sidebar reads the newest 20 chats, then each new one. - **`create_wait`.** A new chat's tab can start to read before the init's creates. - **Resume from `next_offset`.** Each item that the server sends carries its read's `next_offset`. The tab saves it, and a reload or a reconnect sends it back. +- **Opaque offsets.** A `StreamOffset` is an opaque position. The server passes each one to the + browser as its string form, and parses a resume offset back with `StreamOffset.parse`. - **`Superseded` withdrawal.** The page uses `withdrawn_offsets` for tokens that the read gave, and the server uses `withdraws()` for tokens that the tab kept from an earlier read. - **Close with `async with`.** The share snapshot's block in the server, and the sub-agent's @@ -160,9 +164,9 @@ another in one chat, so a new viewer sees each case. snapshot's offset, and an unknown snapshot gives `StreamNotFoundError`. - **Error classes.** `StreamClosedError` and `StreamAlreadyExistsError` in `share`, and `StreamNotFoundError` in `share_info`. -- **Payload type.** `[Turn]`, `[Token]`, `[SharedTurn]` and `[SessionInfo]` on the creates and - gets, and `payload_type=` on `client.to_stream`. -- **`client.get_stream[T]` by id for a sub-agent.** The server reads a sub-agent's Streams by the +- **Payload type.** `payload_type=Turn`, `payload_type=Token`, `payload_type=SharedTurn` and + `payload_type=SessionInfo` on the creates and the id-form gets. +- **`client.get_stream` by id for a sub-agent.** The server reads a sub-agent's Streams by the ids in its Turn, for the page's panel. - **In-memory backend.** The tests run on it and on Redis. The server uses it when `CHAT_APP_REDIS_URL` isn't set. @@ -170,7 +174,7 @@ another in one chat, so a new viewer sees each case. Activities, each model or tool attempt is an Activity Producer, a sub-agent writes through its own Local Activities, and the server writes as a client. -The app doesn't use `Stream.producer()`, `SyncStream`, `close_nowait` or `StreamFencedError`. +The app doesn't use `streams.producer()`, `SyncStream`, `close_nowait` or `StreamFencedError`. ## Coming @@ -178,7 +182,6 @@ These are approved but not built yet, so the app doesn't use them. - A `closed` item with a close status. The log would show why a Stream closed. - `max_record_age` on a read. -- Opaque offsets. The app already passes offsets to the browser as strings. The Python facade has no public `describe` yet, so the app uses only `get_offset`. diff --git a/tests/contrib/streams/examples/chat_app/activities.py b/tests/contrib/streams/examples/chat_app/activities.py index e8ce325a34..6087d7df94 100644 --- a/tests/contrib/streams/examples/chat_app/activities.py +++ b/tests/contrib/streams/examples/chat_app/activities.py @@ -7,7 +7,7 @@ from collections.abc import AsyncIterator from temporalio import activity -from temporalio.client import BufferedPublisher +from temporalio.client import BufferedAppender from temporalio.exceptions import ApplicationError from .models import Model, RawContent @@ -39,11 +39,11 @@ def __init__(self, model: Model) -> None: @activity.defn async def call_model(self, input: ModelInput) -> ModelReply: # A Stream from activity.client() writes as this attempt, so a retry withdraws its Records. - tokens = activity.client().to_stream(input.tokens) + tokens = activity.client().get_stream(input.tokens) fail = input.fail_first_attempt and activity.info().attempt == 1 reply = ModelReply("") text: list[str] = [] - async with heartbeats(), BufferedPublisher(tokens) as publisher: + async with heartbeats(), BufferedAppender(tokens) as publisher: streamed = self._model.stream(input.messages, input.tools) async with contextlib.aclosing(streamed) as events: async for event in events: @@ -52,7 +52,7 @@ async def call_model(self, input: ModelInput) -> ModelReply: elif isinstance(event, RawContent): reply.raw = event.blocks else: - publisher.publish(Token(input.number, event, input.round)) + publisher.append(Token(input.number, event, input.round)) text.append(event) if fail and len(text) == FAIL_AFTER: break diff --git a/tests/contrib/streams/examples/chat_app/docs.py b/tests/contrib/streams/examples/chat_app/docs.py index 23510a3a05..cf79f04e65 100644 --- a/tests/contrib/streams/examples/chat_app/docs.py +++ b/tests/contrib/streams/examples/chat_app/docs.py @@ -26,9 +26,9 @@ ), ( "Retention", - "A Record stays readable for the Stream's Retention, also while the Stream is open. A " + "A Record stays readable for the backend's Retention, also while the Stream is open. A " "closed Stream is kept for one Retention after its close. Token deltas matter only to " - "live readers, so a chat gives its tokens Stream a short Retention, such as one hour.", + "live readers, so a backend that stores a chat's tokens can keep a short Retention.", ), ( "Close", @@ -45,7 +45,7 @@ ( "Continue-as-new", "A run that continues as new forwards the refs of its Streams to the next run. The next " - "run gets the Streams with to_stream and creates nothing, so readers see one Stream.", + "run gets the Streams with get_stream and creates nothing, so readers see one Stream.", ), ( "Reading", diff --git a/tests/contrib/streams/examples/chat_app/models/fake.py b/tests/contrib/streams/examples/chat_app/models/fake.py index d873977864..21ed029908 100644 --- a/tests/contrib/streams/examples/chat_app/models/fake.py +++ b/tests/contrib/streams/examples/chat_app/models/fake.py @@ -16,7 +16,7 @@ CANNED_REPLIES = [ "I'm the scripted model, so this demo runs with no API key. Each word that you see is a " - "Record of the tokens Stream. The model Activity writes them through a BufferedPublisher, " + "Record of the tokens Stream. The model Activity writes them through a BufferedAppender, " "which sends them in small batches.", "The Workflow publishes each turn to the turns Stream, and the Activity publishes the tokens " "to a Stream of their own. The browser reads both through the server, which sends each read " diff --git a/tests/contrib/streams/examples/chat_app/server.py b/tests/contrib/streams/examples/chat_app/server.py index 1f985958b3..e2de2d3717 100644 --- a/tests/contrib/streams/examples/chat_app/server.py +++ b/tests/contrib/streams/examples/chat_app/server.py @@ -90,7 +90,11 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: stream_backends=[backend], ) # Each server start sends this create, so it gets the lobby that an earlier start made. - lobby = await client.create_or_get_stream[SessionInfo](LOBBY_ID) + lobby = await client.create_stream( + LOBBY_ID, + payload_type=SessionInfo, + id_conflict_policy=streams.StreamIDConflictPolicy.USE_EXISTING, + ) model = pick_model() task_queue = "streams-chat-app" async with Worker( @@ -145,7 +149,7 @@ def snapshot_of(share_id: str, chat: ChatDep) -> Stream[SharedTurn]: # A share link names a Stream id, so the server reads only the ids that it writes. if not re.fullmatch(SHARE_ID, share_id): raise HTTPException(404, f"There is no snapshot {share_id}") - return chat.client.get_stream[SharedTurn](share_id) + return chat.client.get_stream(share_id, payload_type=SharedTurn) SnapshotDep = Annotated[Stream[SharedTurn], Depends(snapshot_of)] @@ -178,7 +182,7 @@ async def open_session(body: OpenRequest, chat: ChatDep) -> dict[str, Any]: id_reuse_policy=WorkflowIDReusePolicy.REJECT_DUPLICATE, ) started = datetime.now(timezone.utc).isoformat(timespec="seconds") - await chat.client.to_stream(chat.lobby).publish( + await chat.client.get_stream(chat.lobby).append( SessionInfo(session.id, started) ) except WorkflowAlreadyStartedError: @@ -250,29 +254,34 @@ async def end(session: SessionDep) -> dict[str, Any]: @app.post("/api/sessions/{session_id}/share") async def share(session_id: str, refs: StreamsDep, chat: ChatDep) -> dict[str, Any]: - turns = chat.client.to_stream(refs.turns, payload_type=Turn) + turns = chat.client.get_stream( + refs.turns.id, + payload_type=Turn, + owner=refs.turns.owner, + backend=refs.turns.backend, + ) # The snapshot holds the chat up to this offset, which names it. upto = await turns.get_offset() share_id = f"{session_id}-snapshot-{upto}" - shown: dict[int, SharedTurn] = {} - async with contextlib.aclosing(turns.subscribe(wait_new_records=False)) as read: + shown: dict[str, SharedTurn] = {} + async with contextlib.aclosing(turns.read(wait_new_records=False)) as read: async for item in read: if item.offset >= upto: break match item.kind: case streams.ItemKind.SUPERSEDED: for offset in item.withdrawn_offsets: - del shown[offset] + del shown[str(offset)] case streams.ItemKind.RECORD: - shown[item.offset] = SharedTurn( - item.data, item.offset, source_producer(item.producer) + shown[str(item.offset)] = SharedTurn( + item.data, str(item.offset), source_producer(item.producer) ) try: # A strict create, since a share id names one snapshot. - async with await chat.client.create_stream[SharedTurn]( - share_id, retention=timedelta(days=1) + async with await chat.client.create_stream( + share_id, payload_type=SharedTurn ) as snapshot: - await snapshot.publish(*shown.values()) + await snapshot.append(*shown.values()) except streams.StreamClosedError: pass # This point of the chat has a snapshot, and its block closed it. except streams.StreamAlreadyExistsError: @@ -305,7 +314,7 @@ async def share_info(share_id: str, chat: ChatDep) -> dict[str, Any]: @app.get("/api/shares/{share_id}/events", response_class=EventSourceResponse) async def share_events(snapshot: SnapshotDep) -> AsyncIterable[ServerSentEvent]: # A snapshot is closed, so the read ends after its last Turn. - async for event in events("turns", snapshot.subscribe()): + async for event in events("turns", snapshot.read()): yield event @@ -318,8 +327,8 @@ async def agent_events( re.fullmatch(AGENT_STREAM_ID, turns) and re.fullmatch(AGENT_STREAM_ID, tokens) ): raise HTTPException(404, "There is no such sub-agent") - turns_read = chat.client.get_stream[Turn](turns).subscribe() - tokens_read = chat.client.get_stream[Token](tokens).subscribe() + turns_read = chat.client.get_stream(turns, payload_type=Turn).read() + tokens_read = chat.client.get_stream(tokens, payload_type=Token).read() merged = merge(events("sub-turns", turns_read), events("sub-tokens", tokens_read)) async for event in merged: yield event @@ -343,14 +352,25 @@ async def session_events( # The tab sends the offsets that it saved, so a reload or a reconnect resumes there. The # init's create can still be in flight when a new chat's tab starts to read. wait = timedelta(seconds=30) - turns_stream = chat.client.to_stream(refs.turns, payload_type=Turn) - turns_read = turns_stream.subscribe( - offset=int(turns) if turns else None, create_wait=wait + turns_stream = chat.client.get_stream( + refs.turns.id, + payload_type=Turn, + owner=refs.turns.owner, + backend=refs.turns.backend, + ) + turns_read = turns_stream.read( + offset=streams.StreamOffset.parse(turns) if turns else None, create_wait=wait + ) + tokens_stream = chat.client.get_stream( + refs.tokens.id, + payload_type=Token, + owner=refs.tokens.owner, + backend=refs.tokens.backend, ) - tokens_stream = chat.client.to_stream(refs.tokens, payload_type=Token) # A tab that joins reads only new tokens, since the turns Stream has the earlier replies. - tokens_read = tokens_stream.subscribe( - offset=int(tokens) if tokens else streams.END, create_wait=wait + tokens_read = tokens_stream.read( + offset=streams.StreamOffset.parse(tokens) if tokens else streams.END, + create_wait=wait, ) merged = merge( events("lobby", lobby_read(chat)), @@ -361,10 +381,15 @@ async def session_events( yield event -def lobby_read(chat: Chat) -> streams.Subscription[SessionInfo]: - lobby = chat.client.to_stream(chat.lobby, payload_type=SessionInfo) +def lobby_read(chat: Chat) -> streams.Reader[SessionInfo]: + lobby = chat.client.get_stream( + chat.lobby.id, + payload_type=SessionInfo, + owner=chat.lobby.owner, + backend=chat.lobby.backend, + ) # The newest 20 chats, then each new one. - return lobby.subscribe(last=20) + return lobby.read(last=20) async def merge( @@ -389,7 +414,7 @@ async def pump(source: AsyncIterator[ServerSentEvent]) -> None: async def events( name: str, - read: streams.Subscription[Any], + read: streams.Reader[Any], kept: Sequence[streams.Producer] = (), ) -> AsyncIterator[ServerSentEvent]: try: @@ -438,10 +463,8 @@ def plain(value: Any) -> Any: def data_json(value: Any) -> dict[str, Any]: - data = plain(dataclasses.asdict(value)) - if isinstance(value, SharedTurn): - data["source_offset"] = str(value.source_offset) - return data + # A SharedTurn already holds its source offset as the opaque offset's string form. + return plain(dataclasses.asdict(value)) def producer_json(producer: streams.Producer | None) -> dict[str, Any] | None: diff --git a/tests/contrib/streams/examples/chat_app/shared.py b/tests/contrib/streams/examples/chat_app/shared.py index f6a5c060d0..0549dec781 100644 --- a/tests/contrib/streams/examples/chat_app/shared.py +++ b/tests/contrib/streams/examples/chat_app/shared.py @@ -168,10 +168,11 @@ class SharedTurn: """A Record of a snapshot, a copy of a Turn and where the Turn was in the chat's turns Stream. The snapshot is a new Stream that the server writes, so its own offsets and Producer differ. + The source offset is the opaque offset's string form, which a reader parses back. """ turn: Turn - source_offset: int + source_offset: str source_producer: SourceProducer | None = None diff --git a/tests/contrib/streams/examples/chat_app/static/index.html b/tests/contrib/streams/examples/chat_app/static/index.html index 00b0dc23ed..1fa086e67a 100644 --- a/tests/contrib/streams/examples/chat_app/static/index.html +++ b/tests/contrib/streams/examples/chat_app/static/index.html @@ -1069,7 +1069,7 @@

Stream log

async function send(text, fail) { if (busy()) { await api("POST", `/api/sessions/${sessionId}/add`, { text }); - $("sent").textContent = "The add_message Signal published the message with publish_nowait"; + $("sent").textContent = "The add_message Signal appended the message with append_nowait"; return; } const result = await api("POST", `/api/sessions/${sessionId}/messages`, { @@ -1079,7 +1079,7 @@

Stream log

if (!main.seen.has(`turns:record:${result.offset}`)) awaited.add(result.offset); showComposer(); const offset = shortOffset(result.offset); - $("sent").textContent = `The Update's awaited publish returned offset ${offset}`; + $("sent").textContent = `The Update's awaited append returned offset ${offset}`; $("sent").title = `offset ${result.offset}`; } diff --git a/tests/contrib/streams/examples/chat_app/test_chat_app.py b/tests/contrib/streams/examples/chat_app/test_chat_app.py index 2d2c461ed2..5dc327784b 100644 --- a/tests/contrib/streams/examples/chat_app/test_chat_app.py +++ b/tests/contrib/streams/examples/chat_app/test_chat_app.py @@ -52,7 +52,7 @@ async def start(client: Client, worker: Worker) -> WorkflowHandle[ChatSession, N async def read_until_answer( - read: streams.Subscription[Turn], number: int + read: streams.Reader[Turn], number: int ) -> list[streams.Record[Turn]]: records = [] async for item in read: @@ -70,7 +70,12 @@ async def answer( handle = await start(client, worker) refs = await handle.query(ChatSession.streams) await handle.execute_update(ChatSession.send, UserMessage(text)) - read = client.to_stream(refs.turns, payload_type=Turn).subscribe() + read = client.get_stream( + refs.turns.id, + payload_type=Turn, + owner=refs.turns.owner, + backend=refs.turns.backend, + ).read() records = await read_until_answer(read, 1) await read.aclose() return handle, records @@ -84,13 +89,23 @@ async def test_turn_and_its_tokens(stream_client: Client): await handle.signal(ChatSession.end) await handle.result() # The session closed both Streams, so each read ends. - turns = stream_client.to_stream(refs.turns, payload_type=Turn) - records = [r async for r in turns.subscribe() if isinstance(r, streams.Record)] - tokens = stream_client.to_stream(refs.tokens, payload_type=Token) - deltas = [r.data async for r in tokens.subscribe() if isinstance(r, streams.Record)] + turns = stream_client.get_stream( + refs.turns.id, + payload_type=Turn, + owner=refs.turns.owner, + backend=refs.turns.backend, + ) + records = [r async for r in turns.read() if isinstance(r, streams.Record)] + tokens = stream_client.get_stream( + refs.tokens.id, + payload_type=Token, + owner=refs.tokens.owner, + backend=refs.tokens.backend, + ) + deltas = [r.data async for r in tokens.read() if isinstance(r, streams.Record)] system, user, assistant, ended = records assert system.data == Turn("system", "Run 1 of the session started", 0) - assert user.data == Turn("user", "hello", 1) and user.offset == offset + assert user.data == Turn("user", "hello", 1) and str(user.offset) == offset assert assistant.data.text.startswith('You said "hello".') assert (assistant.data.round, assistant.data.final) == (1, True) assert "".join(d.text for d in deltas) == assistant.data.text @@ -155,7 +170,12 @@ async def test_message_during_a_run_joins_the_next_round(stream_client: Client): await handle.execute_update( ChatSession.send, UserMessage("Search the docs for retention.") ) - read = stream_client.to_stream(refs.turns, payload_type=Turn).subscribe() + read = stream_client.get_stream( + refs.turns.id, + payload_type=Turn, + owner=refs.turns.owner, + backend=refs.turns.backend, + ).read() records = [] async for item in read: if isinstance(item, streams.Record): @@ -196,10 +216,13 @@ async def test_sub_agent_writes_its_own_streams(stream_client: Client): [link] = [r.data for r in records if r.data.role == "agent"] assert link.agent is not None # The sub-agent closed its Streams, so each read ends. - child = stream_client.to_stream(link.agent.turns, payload_type=Turn) - child_turns = [ - r.data async for r in child.subscribe() if isinstance(r, streams.Record) - ] + child = stream_client.get_stream( + link.agent.turns.id, + payload_type=Turn, + owner=link.agent.turns.owner, + backend=link.agent.turns.backend, + ) + child_turns = [r.data async for r in child.read() if isinstance(r, streams.Record)] assert [(t.role, t.tool) for t in child_turns] == [ ("user", None), ("assistant", None), @@ -208,8 +231,13 @@ async def test_sub_agent_writes_its_own_streams(stream_client: Client): ("tool_result", "search_docs"), ("assistant", None), ] - tokens = stream_client.to_stream(link.agent.tokens, payload_type=Token) - assert [r async for r in tokens.subscribe() if isinstance(r, streams.Record)] + tokens = stream_client.get_stream( + link.agent.tokens.id, + payload_type=Token, + owner=link.agent.tokens.owner, + backend=link.agent.tokens.backend, + ) + assert [r async for r in tokens.read() if isinstance(r, streams.Record)] assert records[-1].data.text.startswith("The sub-agent reports back.") @@ -225,8 +253,13 @@ async def failed_attempt( ) await handle.signal(ChatSession.end) await handle.result() - tokens = client.to_stream(refs.tokens, payload_type=Token) - return refs, [item async for item in tokens.subscribe()] + tokens = client.get_stream( + refs.tokens.id, + payload_type=Token, + owner=refs.tokens.owner, + backend=refs.tokens.backend, + ) + return refs, [item async for item in tokens.read()] async def test_retry_withdraws_the_failed_attempt(stream_client: Client): @@ -252,10 +285,22 @@ async def test_server_checks_kept_tokens_with_withdraws(stream_client: Client): producer = server.producer_json(shown[0].producer) assert producer is not None kept = server.kept_producers(json.dumps([producer["wire"]])) - tokens = stream_client.to_stream(refs.tokens, payload_type=Token) + tokens = stream_client.get_stream( + refs.tokens.id, + payload_type=Token, + owner=refs.tokens.owner, + backend=refs.tokens.backend, + ) + # Read the shown tokens again to get where the tab's read resumes, as it would from the + # next_offset that it saved. + resume_read = tokens.read(wait_new_records=False) + for _ in range(len(shown)): + await anext(resume_read) + resume = resume_read.next_offset + await resume_read.aclose() [superseded] = [ server.item_json(item, kept) - async for item in tokens.subscribe(offset=shown[-1].offset + 1) + async for item in tokens.read(offset=resume) if isinstance(item, streams.Superseded) ] assert superseded["withdrawn_offsets"] == [] @@ -266,9 +311,14 @@ async def test_resume_from_a_saved_offset(stream_client: Client): async with chat_worker(stream_client) as worker: handle = await start(stream_client, worker) refs = await handle.query(ChatSession.streams) - turns = stream_client.to_stream(refs.turns, payload_type=Turn) + turns = stream_client.get_stream( + refs.turns.id, + payload_type=Turn, + owner=refs.turns.owner, + backend=refs.turns.backend, + ) await handle.execute_update(ChatSession.send, UserMessage("message 1")) - read = turns.subscribe() + read = turns.read() await read_until_answer(read, 1) saved = read.next_offset await read.aclose() @@ -279,7 +329,7 @@ async def test_resume_from_a_saved_offset(stream_client: Client): await handle.execute_update( ChatSession.send, UserMessage(f"message {number}") ) - resumed = turns.subscribe(offset=saved) + resumed = turns.read(offset=saved) later = [r.data for r in await read_until_answer(resumed, last)] await resumed.aclose() assert await handle.query(ChatSession.streams) == refs @@ -292,7 +342,9 @@ async def test_resume_from_a_saved_offset(stream_client: Client): async def test_share_again_is_a_strict_create_of_an_existing_id(stream_client: Client): async with chat_worker(stream_client) as worker: - lobby = await stream_client.create_stream[SessionInfo](f"lobby-{uuid.uuid4()}") + lobby = await stream_client.create_stream( + f"lobby-{uuid.uuid4()}", payload_type=SessionInfo + ) server.app.state.chat = server.Chat( stream_client, lobby.ref, worker.task_queue, "test", "fake", "scripted" ) @@ -306,13 +358,18 @@ async def test_share_again_is_a_strict_create_of_an_existing_id(stream_client: C await http.post(f"/api/sessions/{session_id}/messages", json={"text": "hi"}) handle = stream_client.get_workflow_handle(session_id) refs = await handle.query(ChatSession.streams) - turns = stream_client.to_stream(refs.turns, payload_type=Turn) - read = turns.subscribe() + turns = stream_client.get_stream( + refs.turns.id, + payload_type=Turn, + owner=refs.turns.owner, + backend=refs.turns.backend, + ) + read = turns.read() await read_until_answer(read, 1) await read.aclose() chat = [ r - async for r in turns.subscribe(wait_new_records=False) + async for r in turns.read(wait_new_records=False) if isinstance(r, streams.Record) ] first = (await http.post(f"/api/sessions/{session_id}/share")).json() @@ -328,12 +385,10 @@ async def test_share_again_is_a_strict_create_of_an_existing_id(stream_client: C assert info.json()["session_id"] == session_id assert outdated.json()["outdated"] is True # The read ends at the snapshot's close. Each copy names its Turn's offset in the chat. - snapshot = stream_client.get_stream[SharedTurn](first["share_id"]) - copies = [ - r.data async for r in snapshot.subscribe() if isinstance(r, streams.Record) - ] + snapshot = stream_client.get_stream(first["share_id"], payload_type=SharedTurn) + copies = [r.data async for r in snapshot.read() if isinstance(r, streams.Record)] assert [c.turn for c in copies] == [r.data for r in chat] - assert [c.source_offset for c in copies] == [r.offset for r in chat] + assert [c.source_offset for c in copies] == [str(r.offset) for r in chat] assert {c.source_producer.kind for c in copies if c.source_producer} == { "local activity" } diff --git a/tests/contrib/streams/examples/chat_app/tools.py b/tests/contrib/streams/examples/chat_app/tools.py index d5175b6a81..95cd6b27a6 100644 --- a/tests/contrib/streams/examples/chat_app/tools.py +++ b/tests/contrib/streams/examples/chat_app/tools.py @@ -149,14 +149,14 @@ def __init__(self, work_scale: float = 1.0) -> None: async def run_tool(self, input: ToolInput) -> ToolResult: call = input.call # A Stream from activity.client() writes as this attempt, so each tool is a Producer. - turns = activity.client().to_stream(input.turns) + turns = activity.client().get_stream(input.turns) def turn(role: Any, text: str) -> Turn: return Turn( role, text, input.number, input.round, tool=call.name, call_id=call.id ) - await turns.publish(turn("tool_progress", f"{call.name} started")) + await turns.append(turn("tool_progress", f"{call.name} started")) await asyncio.sleep(WORK_SECONDS.get(call.name, 0) * self._work_scale) tool = TOOLS.get(call.name) try: @@ -166,5 +166,5 @@ def turn(role: Any, text: str) -> Turn: result = ToolResult(call.id, call.name, tool(call.args)) except (ValueError, SyntaxError, ZeroDivisionError) as err: result = ToolResult(call.id, call.name, f"error: {err}", error=True) - await turns.publish(turn("tool_result", result.text)) + await turns.append(turn("tool_result", result.text)) return result diff --git a/tests/contrib/streams/examples/chat_app/workflow.py b/tests/contrib/streams/examples/chat_app/workflow.py index e1e58602a7..2a40170ee4 100644 --- a/tests/contrib/streams/examples/chat_app/workflow.py +++ b/tests/contrib/streams/examples/chat_app/workflow.py @@ -5,7 +5,7 @@ import asyncio from datetime import timedelta -from temporalio import workflow +from temporalio import streams, workflow from temporalio.common import RetryPolicy from temporalio.exceptions import ActivityError, ApplicationError, ChildWorkflowError @@ -101,13 +101,13 @@ async def answer(self, number: int, fail_first_attempt: bool = False) -> str: if final: # A message that comes from here on starts a new turn. self.answering = False - await self.turns.publish( + await self.turns.append( Turn("assistant", reply.text, number, round, final=final) ) if final: return reply.text if reply.tool_calls: - await self.turns.publish( + await self.turns.append( *( Turn( "tool_call", @@ -136,7 +136,7 @@ async def _stop(self, turn: Turn) -> str: # The next turn's model still sees a message that came during this one. self.history += [Message("user", m.text) for m in self.added] self.added.clear() - await self.turns.publish(turn) + await self.turns.append(turn) return "" async def _call( @@ -163,7 +163,7 @@ async def _call( except (ActivityError, ChildWorkflowError) as err: # The tool didn't write its result, so the Workflow writes the failure. result = ToolResult(call.id, call.name, f"error: {notice(err)}", error=True) - await self.turns.publish( + await self.turns.append( Turn( "tool_result", result.text, @@ -186,20 +186,24 @@ def __init__(self, input: SessionInput) -> None: if input.streams is None: # A chat id can come back, such as in a restarted session, so the run gets the # chat's Streams if they exist. - turns = workflow.create_or_get_stream_nowait[Turn]( - turns_id(input.session_id) + turns = workflow.create_stream_nowait( + turns_id(input.session_id), + payload_type=Turn, + id_conflict_policy=streams.StreamIDConflictPolicy.USE_EXISTING, ) - # Token deltas matter only to live readers, so they get a short Retention. - tokens = workflow.create_or_get_stream_nowait[Token]( - tokens_id(input.session_id), retention=timedelta(hours=1) + # Retention is a backend setting now, so the tokens Stream uses the backend's. + tokens = workflow.create_stream_nowait( + tokens_id(input.session_id), + payload_type=Token, + id_conflict_policy=streams.StreamIDConflictPolicy.USE_EXISTING, ) else: - turns = workflow.to_stream(input.streams.turns) - tokens = workflow.to_stream(input.streams.tokens) + turns = workflow.get_stream(input.streams.turns) + tokens = workflow.get_stream(input.streams.tokens) self._turns = turns self._agent = Agent(turns, tokens, list(TOOL_SPECS), list(input.history)) # The init can't await, so it publishes with the _nowait call. - turns.publish_nowait( + turns.append_nowait( Turn("system", f"Run {input.run} of the session started", self._number) ) @@ -208,13 +212,14 @@ def streams(self) -> SessionStreams: return SessionStreams(self._turns.ref, self._agent.tokens.ref) @workflow.update - async def send(self, message: UserMessage) -> int: + async def send(self, message: UserMessage) -> str: self._number += 1 number = self._number - # The awaited publish gives the Record's offset, which the server hands to the browser. - offset = await self._turns.publish(Turn("user", message.text, number)) + # The awaited append gives the Record's offset, which the server hands to the browser as + # its opaque string form. + offset = await self._turns.append(Turn("user", message.text, number)) self._waiting.append(Waiting(number, message)) - return offset + return str(offset) @send.validator def check_send(self, message: UserMessage) -> None: @@ -236,8 +241,8 @@ def add_message(self, message: UserMessage) -> None: self._number += 1 number = self._number self._waiting.append(Waiting(number, message)) - # A signal handler can't await, so the user's Turn goes out with publish_nowait. - self._turns.publish_nowait(Turn("user", message.text, number)) + # A signal handler can't await, so the user's Turn goes out with append_nowait. + self._turns.append_nowait(Turn("user", message.text, number)) @workflow.signal def end(self) -> None: @@ -278,7 +283,7 @@ async def run(self, input: SessionInput) -> None: waiting=self._waiting, ) ) - await self._turns.publish(Turn("system", "The chat ended", self._number)) + await self._turns.append(Turn("system", "The chat ended", self._number)) # A run's end doesn't close a Stream, so the session closes both. await self._turns.close() await agent.tokens.close() @@ -294,13 +299,11 @@ async def run(self, input: ResearchInput) -> ToolResult: # The ids include this run's id, which the parent can't know. So the sub-agent creates # its Streams itself, and hands their refs to the parent in a Turn. own_id = f"{info.workflow_id}-{info.run_id[:8]}" - turns = await workflow.create_stream[Turn](turns_id(own_id)) - tokens = await workflow.create_stream[Token]( - tokens_id(own_id), retention=timedelta(hours=1) - ) + turns = await workflow.create_stream(turns_id(own_id), payload_type=Turn) + tokens = await workflow.create_stream(tokens_id(own_id), payload_type=Token) call = input.call - parent = workflow.to_stream(input.parent_turns) - await parent.publish( + parent = workflow.get_stream(input.parent_turns) + await parent.append( Turn( "agent", f"A sub-agent researches {input.topic}", @@ -314,13 +317,13 @@ async def run(self, input: ResearchInput) -> ToolResult: # The block end closes both Streams, so a reader of the sub-agent sees it end. async with turns, tokens: prompt = f"Research {input.topic}" - await turns.publish(Turn("user", prompt, 1)) + await turns.append(Turn("user", prompt, 1)) agent = Agent(turns, tokens, ["search_docs"], [Message("user", prompt)]) summary = await agent.answer(1) result = ToolResult( call.id, call.name, summary or "The sub-agent found nothing." ) - await parent.publish( + await parent.append( Turn( "tool_result", result.text, diff --git a/tests/contrib/streams/examples/client_creates_stream.py b/tests/contrib/streams/examples/client_creates_stream.py index 796366c9fc..9bc7726b23 100644 --- a/tests/contrib/streams/examples/client_creates_stream.py +++ b/tests/contrib/streams/examples/client_creates_stream.py @@ -2,8 +2,9 @@ The client creates the Stream with its settings, passes its ref to the Workflow as data, reads it, and closes it at the end of ``async with``. The create fails if the id exists, and -``create_or_get_stream`` would get the Stream instead. The Workflow gets the ref as data, turns -it into a Stream to write, and passes the same ref to an Activity. +``id_conflict_policy=StreamIDConflictPolicy.USE_EXISTING`` would get the Stream instead. The +Workflow gets the ref as data, turns it into a Stream to write, and passes the same ref to an +Activity. """ from __future__ import annotations @@ -37,12 +38,12 @@ class Chat: @workflow.run async def run(self, ref: streams.StreamRef[Turn]) -> None: # The Workflow only writes. The client closes the Stream. - turns = workflow.to_stream(ref) - turns.publish(Turn("model", "hi")) + turns = workflow.get_stream(ref) + turns.append(Turn("model", "hi")) await workflow.execute_activity( notify_user, ref, start_to_close_timeout=SECONDS_30 ) - await turns.publish(Turn("last", "bye")) + await turns.append(Turn("last", "bye")) def is_last_turn(r: streams.Record[Turn]) -> bool: @@ -56,8 +57,8 @@ async def main(client: Client, task_queue: str) -> list[Turn]: chat_id = f"chat-{uuid.uuid4()}" turns: list[Turn] = [] render = turns.append - # create_stream raises if the id exists, and create_or_get_stream gets it instead. - stream = await client.create_stream[Turn](chat_id, retention=timedelta(days=1)) + # create_stream raises if the id exists; USE_EXISTING would get it instead. + stream = await client.create_stream(chat_id, payload_type=Turn) async with stream: # optional, the block end closes it await client.start_workflow( Chat.run, @@ -65,7 +66,7 @@ async def main(client: Client, task_queue: str) -> list[Turn]: id=chat_id, task_queue=task_queue, ) - async for r in stream.subscribe(): + async for r in stream.read(): if r.kind != streams.ItemKind.RECORD: continue render(r.data) diff --git a/tests/contrib/streams/examples/resume_from_offset.py b/tests/contrib/streams/examples/resume_from_offset.py index e637244324..1a017216ec 100644 --- a/tests/contrib/streams/examples/resume_from_offset.py +++ b/tests/contrib/streams/examples/resume_from_offset.py @@ -27,10 +27,10 @@ def __init__(self) -> None: @workflow.run async def run(self, ref: streams.StreamRef[Turn], first: list[str]) -> None: - turns = workflow.to_stream(ref) - await turns.publish(*map(Turn, first)) + turns = workflow.get_stream(ref) + await turns.append(*map(Turn, first)) await workflow.wait_condition(lambda: bool(self.more)) - await turns.publish(*map(Turn, self.more)) + await turns.append(*map(Turn, self.more)) @workflow.signal def add(self, texts: list[str]) -> None: @@ -41,12 +41,12 @@ class OffsetStore: """Where the Reader keeps its place, such as a database row.""" def __init__(self) -> None: - self.offsets: dict[str, int] = {} + self.offsets: dict[str, streams.StreamOffset] = {} - def load_offset(self, key: str) -> int | None: + def load_offset(self, key: str) -> streams.StreamOffset | None: return self.offsets.get(key) - def save_offset(self, key: str, offset: int | None) -> None: + def save_offset(self, key: str, offset: streams.StreamOffset | None) -> None: if offset is not None: self.offsets[key] = offset @@ -54,9 +54,9 @@ def save_offset(self, key: str, offset: int | None) -> None: async def read_some( client: Client, chat_id: str, store: OffsetStore, count: int ) -> list[str]: - stream = client.get_stream[Turn](chat_id) + stream = client.get_stream(chat_id, payload_type=Turn) seen = [] - read = stream.subscribe(offset=store.load_offset("turns")) + read = stream.read(offset=store.load_offset("turns")) async for r in read: if r.kind != streams.ItemKind.RECORD: continue @@ -70,10 +70,10 @@ async def read_some( async def read_new(client: Client, chat_id: str) -> list[str]: # A view that shows only what comes next starts after the newest Record. - stream = client.get_stream[Turn](chat_id) + stream = client.get_stream(chat_id, payload_type=Turn) return [ r.data.text - async for r in stream.subscribe(offset=streams.END, wait_new_records=False) + async for r in stream.read(offset=streams.END, wait_new_records=False) if isinstance(r, streams.Record) ] @@ -83,7 +83,7 @@ async def main( ) -> tuple[list[str], list[str], list[str]]: async with Worker(client, task_queue=task_queue, workflows=[Chat]): chat_id = f"chat-{uuid.uuid4()}" - async with await client.create_stream[Turn](chat_id) as turns: + async with await client.create_stream(chat_id, payload_type=Turn) as turns: handle = await client.start_workflow( Chat.run, args=[turns.ref, ["a", "b"]], diff --git a/tests/contrib/streams/examples/test_examples.py b/tests/contrib/streams/examples/test_examples.py index 4c6e9b2f40..137b4d5dea 100644 --- a/tests/contrib/streams/examples/test_examples.py +++ b/tests/contrib/streams/examples/test_examples.py @@ -58,7 +58,7 @@ async def test_activity_streams_tokens(stream_client: Client): "three", "summary: ONE TWO THREE", ] - assert summary == records[-1].offset + assert summary == str(records[-1].offset) async def test_worker_setup_on_memory(client: Client): diff --git a/tests/contrib/streams/examples/ui_backend_reads.py b/tests/contrib/streams/examples/ui_backend_reads.py index 978976ac1b..ecf43bcce1 100644 --- a/tests/contrib/streams/examples/ui_backend_reads.py +++ b/tests/contrib/streams/examples/ui_backend_reads.py @@ -33,18 +33,18 @@ async def call_model(prompt: str) -> str: class Agent: @workflow.run async def run(self, ref: streams.StreamRef[Turn], prompt: str) -> str: - turns = workflow.to_stream(ref) - await turns.publish(Turn("model")) + turns = workflow.get_stream(ref) + await turns.append(Turn("model")) reply = await workflow.execute_activity( call_model, prompt, start_to_close_timeout=timedelta(minutes=5) ) - await turns.publish(Turn("done", reply)) + await turns.append(Turn("done", reply)) return reply async def chat_server(client: Client, chat_id: str, task_queue: str) -> None: # The block end closes the Stream after the agent's done Turn. - async with await client.create_stream[Turn](chat_id) as turns: + async with await client.create_stream(chat_id, payload_type=Turn) as turns: await client.execute_workflow( Agent.run, args=[turns.ref, "hello there"], @@ -57,21 +57,21 @@ class Screen: """A fake UI. It keeps what it shows by offset, so a withdrawal can remove items.""" def __init__(self) -> None: - self.shown: dict[int, Turn] = {} + self.shown: dict[streams.StreamOffset, Turn] = {} - def render(self, turn: Turn, offset: int) -> None: + def render(self, turn: Turn, offset: streams.StreamOffset) -> None: self.shown[offset] = turn - def unrender(self, offsets: tuple[int, ...]) -> None: + def unrender(self, offsets: tuple[streams.StreamOffset, ...]) -> None: for offset in offsets: del self.shown[offset] # Where the view goes on. A first view has none, so it reads from the start. -saved: list[int] = [] +saved: list[streams.StreamOffset] = [] -def saved_offset() -> int | None: +def saved_offset() -> streams.StreamOffset | None: return saved[-1] if saved else None @@ -82,8 +82,8 @@ async def main(client: Client, task_queue: str) -> Screen: chat_id = f"chat-{uuid.uuid4()}" server = asyncio.create_task(chat_server(client, chat_id, task_queue)) screen = Screen() - stream = client.get_stream[Turn](chat_id) # by id, with no call - async for r in stream.subscribe( + stream = client.get_stream(chat_id, payload_type=Turn) # by id, with no call + async for r in stream.read( offset=saved_offset(), # wherever the app keeps it, or streams.END create_wait=timedelta(seconds=10), # the creator may not have created it ): diff --git a/tests/contrib/streams/examples/worker_setup.py b/tests/contrib/streams/examples/worker_setup.py index 400b7cdb4a..59ebae8f5f 100644 --- a/tests/contrib/streams/examples/worker_setup.py +++ b/tests/contrib/streams/examples/worker_setup.py @@ -23,10 +23,10 @@ class Greeter: @workflow.init def __init__(self, name: str) -> None: # The init can't await, so it creates and publishes with the _nowait calls. - self.tokens = workflow.create_stream_nowait[str]( - f"{workflow.info().workflow_id}-tokens" + self.tokens = workflow.create_stream_nowait( + f"{workflow.info().workflow_id}-tokens", payload_type=str ) - self.tokens.publish_nowait(f"hello {name}") + self.tokens.append_nowait(f"hello {name}") @workflow.query def stream(self) -> streams.StreamRef[str]: @@ -34,7 +34,7 @@ def stream(self) -> streams.StreamRef[str]: @workflow.run async def run(self, name: str) -> str: - await self.tokens.publish(f"bye {name}") + await self.tokens.append(f"bye {name}") await self.tokens.close() return name @@ -75,10 +75,12 @@ async def main(client: Client, task_queue: str) -> list[str]: Greeter.run, "ada", id=f"greeter-{uuid.uuid4()}", task_queue=task_queue ) await handle.result() - stream = client.to_stream(await handle.query(Greeter.stream)) - return [r.data async for r in stream.subscribe() if isinstance(r, streams.Record)] + stream = client.get_stream(await handle.query(Greeter.stream)) + return [r.data async for r in stream.read() if isinstance(r, streams.Record)] async def create_on_second_backend(client: Client) -> str | None: - stream = await client.create_stream[str](f"chat-{uuid.uuid4()}", backend="redis-b") + stream = await client.create_stream( + f"chat-{uuid.uuid4()}", backend="redis-b", payload_type=str + ) return stream.ref.backend From a3edc2822c1c8b2155840b9f135129318f4216c9 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 8 Nov 2026 14:40:47 -0700 Subject: [PATCH 4/7] Bumped the Core submodule to the owner and withdrawal removal. --- temporalio/bridge/sdk-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index 872c90fcc4..b2987e6a50 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit 872c90fcc4782380c661de2e763484f9f205fc8c +Subproject commit b2987e6a50f01bb373a1da90b3d490cad7b1bfae From 41fa10693e4500f9fccbc68075165df486d507a9 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 8 Nov 2026 14:40:47 -0700 Subject: [PATCH 5/7] Removed owner, regenerated the Stream bindings, made supersede opt-in. --- scripts/_wit/stream-service.wit | 165 ++-- scripts/gen_stream_service.py | 2 +- temporalio/api/enums/v1/__init__.py | 2 + temporalio/api/enums/v1/stream_pb2.py | 26 +- temporalio/api/enums/v1/stream_pb2.pyi | 41 + temporalio/api/stream/v1/__init__.py | 14 +- temporalio/api/stream/v1/message_pb2.py | 145 ++-- temporalio/api/stream/v1/message_pb2.pyi | 424 +++++----- .../streamservice/v1/request_response_pb2.py | 40 +- .../streamservice/v1/request_response_pb2.pyi | 60 +- .../bridge/proto/streams/streams_pb2.pyi | 7 +- temporalio/client/_client.py | 7 +- temporalio/client/_streams.py | 6 +- temporalio/nexus/system/_support.py | 34 +- temporalio/streams/__init__.py | 58 +- temporalio/streams/_access.py | 2 +- temporalio/streams/_activity.py | 7 +- temporalio/streams/_activity_stream.py | 6 +- temporalio/streams/_close.py | 39 + temporalio/streams/_extension.py | 28 +- temporalio/streams/_ids.py | 5 +- temporalio/streams/_items.py | 168 +--- temporalio/streams/_reader.py | 81 +- temporalio/streams/_ref.py | 35 +- temporalio/streams/_stream_client.py | 29 +- temporalio/streams/_workflow.py | 46 +- temporalio/streams/_writer.py | 38 +- temporalio/streams/_writes.py | 63 +- temporalio/streams/superseded.py | 150 ++++ temporalio/streams/system/__init__.py | 48 +- .../streams/system/client/_extensions.py | 8 - .../system/client/_resources/stream.py | 7 +- temporalio/streams/system/models.py | 585 ++++++++------ .../system/workflow/_resources/stream.py | 10 +- .../workflow/operations/create_stream.py | 8 - temporalio/streams/writer_system/__init__.py | 30 +- temporalio/streams/writer_system/models.py | 729 +----------------- temporalio/workflow/_streams.py | 11 +- .../streams/examples/agent_publishes_turns.py | 4 +- .../streams/examples/chat_app/server.py | 117 ++- .../examples/chat_app/test_chat_app.py | 47 +- .../streams/examples/ui_backend_reads.py | 22 +- .../contrib/streams/test_activity_streams.py | 68 +- tests/contrib/streams/test_bindings.py | 25 - tests/contrib/streams/test_facade.py | 4 +- tests/contrib/streams/test_pydantic.py | 2 - tests/contrib/streams/test_reader.py | 37 +- tests/contrib/streams/test_redis.py | 91 +-- tests/contrib/streams/test_refs.py | 118 +-- tests/contrib/streams/test_stream_client.py | 40 +- tests/contrib/streams/test_superseded.py | 144 ++++ tests/contrib/streams/test_vectors.py | 80 +- tests/contrib/streams/test_workflow_module.py | 90 +-- 53 files changed, 1531 insertions(+), 2522 deletions(-) create mode 100644 temporalio/streams/_close.py create mode 100644 temporalio/streams/superseded.py create mode 100644 tests/contrib/streams/test_superseded.py diff --git a/scripts/_wit/stream-service.wit b/scripts/_wit/stream-service.wit index 8dd6309607..c7944a0b01 100644 --- a/scripts/_wit/stream-service.wit +++ b/scripts/_wit/stream-service.wit @@ -14,6 +14,8 @@ world system { interface stream-service { use nexus:temporal-types/model@1.0.0.{ duration, + failure, + payload, placeholder, }; @@ -32,31 +34,10 @@ interface stream-service { record stream-ref { /// @nexus.doc "The Stream id, 1 to 4096 bytes." id: string, - /// @nexus.doc "The run chain or Standalone Activity of a bound Stream. Empty if standalone." - owner: option, /// @nexus.doc "Registered backend name, empty for the default. The Store never reads it." backend: option, } - /// @nexus.proto "temporal.api.enums.v1.ExecutionType" - enum execution-type { - /// @nexus.fallback - execution-type-unspecified, - execution-type-workflow, - execution-type-activity, - execution-type-nexus-operation, - } - - /// @nexus.doc "A Workflow run chain or a Standalone Activity run." - /// @nexus.proto "temporal.api.common.v1.Execution" - record execution { - %type: option, - /// @nexus.doc "The Workflow id, or the Standalone Activity's id." - business-id: option, - /// @nexus.doc "The run chain's first run id, or the Standalone Activity's run id." - run-id: option, - } - /// @nexus.doc "A Workflow Task attempt of the run." /// @nexus.proto "temporal.api.stream.v1.WorkflowTaskEpoch" record workflow-task-epoch { @@ -89,33 +70,53 @@ interface stream-service { attempt: option, } + /// @nexus.doc "A Workflow run Producer. A higher epoch withdraws the run's orphaned Records." + /// @nexus.proto "temporal.api.stream.v1.WorkflowProducer" + record workflow-producer { + /// @nexus.doc "The first execution run id of the run chain." + run-id: option, + epoch: option, + } + + /// @nexus.doc "A Local Activity Producer. A higher epoch withdraws the Producer's lower epochs." + /// @nexus.proto "temporal.api.stream.v1.LocalActivityProducer" + record local-activity-producer { + run-id: option, + epoch: option, + } + + /// @nexus.doc "An Activity Producer. A higher epoch withdraws the Producer's lower epochs." + /// @nexus.proto "temporal.api.stream.v1.ActivityProducer" + record activity-producer { + run-id: option, + epoch: option, + } + + /// @nexus.doc "A client Producer. A higher epoch only fences the lower ones." + /// @nexus.proto "temporal.api.stream.v1.ClientProducer" + record client-producer { + epoch: option, + } + // The case is the kind of the Producer, so a kind sets only its own fields. - /// @nexus.doc "The epoch of each kind of Producer." - /// @nexus.proto "temporal.api.stream.v1.ProducerEpoch.kind" - variant producer-epoch-kind { - workflow(workflow-task-epoch), - local-activity(local-activity-epoch), - activity(activity-epoch), - standalone-activity(activity-epoch), - client(client-epoch), - } - - /// @nexus.doc "The ordered attempt of a Producer. A higher epoch supersedes a lower one." - /// @nexus.proto "temporal.api.stream.v1.ProducerEpoch" - record producer-epoch { - // A newer server can send a kind that this SDK doesn't know. Then the kind is empty. - kind: option, + /// @nexus.doc "The kind of a Producer, with its run and epoch." + /// @nexus.proto "temporal.api.stream.v1.Producer.kind" + variant producer-kind { + workflow(workflow-producer), + local-activity(local-activity-producer), + activity(activity-producer), + standalone-activity(activity-producer), + client(client-producer), } - /// @nexus.doc "The Producer of a batch." + /// @nexus.doc "The code that wrote a batch. Its id is stable across attempts." /// @nexus.proto "temporal.api.stream.v1.Producer" record producer { /// @nexus.doc "Stable across attempts, such as wf: for a Workflow run." id: string, + // A newer server can send a kind that this SDK doesn't know. Then the kind is empty. /// @nexus.doc "Required. Its case gives the kind of the Producer." - epoch: option, - /// @nexus.doc "The run of the Producer, so nobody parses the id. Empty for a client." - run-id: option, + kind: option, } /// @nexus.doc "One Record to append." @@ -126,30 +127,11 @@ interface stream-service { body: body-t, } - // A Local Activity, Activity or Standalone Activity write withdraws the Records of its Producer - // with a lower epoch. A Workflow write, with its task t and the previous completed task c, - // withdraws the Records of the run and of its Local Activities whose Workflow Task is in - // failed-tasks or strictly between c and t. It also withdraws those at the started event id of c - // or of t with a lower epoch than c or t. No task in that range can have completed. A client - // write withdraws nothing. - /// @nexus.doc "The Producer of a write, and which earlier Records the write withdraws." + /// @nexus.doc "The Producer of a write. An append and a close carry the same writer." /// @nexus.proto "temporal.api.stream.v1.StreamWriter" record stream-writer { - /// @nexus.doc "Required on a write. Its epoch's case picks the withdrawal rule." + /// @nexus.doc "Required on a write." producer: option, - /// @nexus.doc "Workflow and Local Activity only. The run's last completed Workflow Task." - previous-completed-task: option, - /// @nexus.doc "Workflow and Local Activity only. The run's failed or timed out Workflow Tasks." - failed-tasks: list, - /// @nexus.doc "Retry run only. The previous run, all of whose Records are withdrawn." - withdraw-run: option, - } - - /// @nexus.doc "Which earlier Records a Superseded item withdraws." - /// @nexus.proto "temporal.api.stream.v1.StreamSuperseded" - record stream-superseded { - /// @nexus.doc "The writer that withdraws. Its epoch is empty when only withdraw-run applies." - writer: option, } /// @nexus.doc "One Record that a Producer appended." @@ -160,12 +142,36 @@ interface stream-service { producer: option, } - // The read builds a Superseded item from a flagged batch. - /// @nexus.doc "A read item: a Record or a Superseded item. A close writes no item." + /// How a Stream closed. Only the owner's end on the server terminates or times out. + /// @nexus.proto "temporal.api.enums.v1.StreamCloseStatus" + enum stream-close-status { + /// @nexus.fallback + stream-close-status-unspecified, + stream-close-status-closed, + stream-close-status-owner-completed, + stream-close-status-owner-failed, + stream-close-status-owner-canceled, + stream-close-status-owner-terminated, + stream-close-status-owner-timed-out, + stream-close-status-idle, + } + + /// @nexus.doc "The close of a Stream. A read gives it last." + /// @nexus.proto "temporal.api.stream.v1.StreamClose" + record stream-close { + /// @nexus.doc "The Producer that closed the Stream." + producer: option, + %result: option, + failure: option, + /// @nexus.doc "Which of result or failure is set, and why the Stream closed." + status: option, + } + + /// @nexus.doc "A read item: a Record or the close." /// @nexus.proto "temporal.api.stream.v1.StreamRecord.item" variant stream-item { data(stream-data), - superseded(stream-superseded), + close(stream-close), } // An offset is ordered and unique in one Stream, and sparse. Offsets have gaps, so they don't @@ -183,24 +189,12 @@ interface stream-service { /// "Create a Stream. If it exists, fail, or with use-existing get the open one." /// @nexus.proto "temporal.api.streamservice.v1.CreateStreamRequest" record create-stream-request { - // Its field name isn't ref, so the created stream takes its ref from the response, which - // carries the owner. + // Its field name isn't ref, so the created Stream takes its ref from the response. /// @nexus.doc "The ref of the Stream to create. Its id is the Stream id." /// @nexus.flatten-in-api stream-ref: stream-ref, /// @nexus.doc "The age after which a Record is dropped. Empty is the Store default." retention: option, - // A wrong run would silently withdraw real Records, so it is never a parameter. The helper - // gives the previous run in a retry run's Workflow, and nothing elsewhere. - /// @nexus.doc "Retry run only. The previous run, whose Records this withdraws." - /// @nexus.source targets="workflow" python="withdraw_run()" typescript="withdrawRun()" - /// dotnet="StreamWorkflowContext.WithdrawRun()" - /// @nexus.source targets="client" python="withdraw_run()" typescript="withdrawRun()" - /// dotnet="StreamWorkflowContext.WithdrawRun()" - withdraw-run: option, - // A bound Stream's owner. An SDK creates only standalone Streams for now. - /// @nexus.omit - owner: placeholder, /// @nexus.doc "What the create does when the Stream exists. Unspecified is fail." id-conflict-policy: option, // A retry of the create that made the Stream sends the same id, so it doesn't fail as a @@ -209,9 +203,6 @@ interface stream-service { /// @nexus.source python="request_id()" typescript="requestId()" /// dotnet="StreamWorkflowContext.RequestId()" request-id: option, - // Native only. The SDK Stores don't read it. - /// @nexus.omit - close-with-run: placeholder, /// @nexus.omit namespace: placeholder, // Native only. The SDK Stores don't read it. @@ -242,8 +233,8 @@ interface stream-service { record append-records-request { ref: stream-ref, records: list, - // A wrong run or task would silently withdraw real Records, so the writer is never a - // parameter. The helper of each context fills only the fields that apply where it runs. + // The writer carries the Producer, which only the context helper sets right, so it is never a + // parameter. The helper of each context fills the Producer that applies where it runs. /// @nexus.doc "Required outside a Workflow. In a Workflow the transport sets it." /// @nexus.source targets="workflow" python="workflow_writer()" /// typescript="workflowWriter()" dotnet="StreamWorkflowContext.Writer()" @@ -257,9 +248,6 @@ interface stream-service { sequence: option, /// @nexus.omit namespace: placeholder, - // Native only. The SDK Stores don't read them. - /// @nexus.omit - defer-withdrawal: placeholder, /// @nexus.omit task-token: placeholder, } @@ -275,7 +263,7 @@ interface stream-service { /// @nexus.proto "temporal.api.streamservice.v1.CloseStreamRequest" record close-stream-request { ref: stream-ref, - // The SDK Stores read only the Producer. The server also reads the withdrawal fields. + // The SDK Stores read only the Producer. /// @nexus.source targets="workflow" python="workflow_writer()" /// typescript="workflowWriter()" dotnet="StreamWorkflowContext.Writer()" /// @nexus.source targets="client" python="client_writer()" typescript="clientWriter()" @@ -307,8 +295,7 @@ interface stream-service { from-offset: option, /// @nexus.doc "Where to start when from-offset is empty. Unspecified is the beginning." start: option, - /// @nexus.doc - /// "About this many items per page, 1 to 1000. A Superseded item stays with its Record." + /// @nexus.doc "About this many items per page, 1 to 1000." max-records: option, /// @nexus.doc "How long to wait for a new Record before an empty answer. At most 60 s." wait: option, @@ -353,7 +340,7 @@ interface stream-service { /// @nexus.proto "temporal.api.streamservice.v1.DescribeStreamResponse" record describe-stream-response { status: option, - /// @nexus.doc "The newest offset, of a Record or a Superseded item. Empty for none." + /// @nexus.doc "The newest offset, of a Record. Empty for none." last-offset: option, retention: option, } diff --git a/scripts/gen_stream_service.py b/scripts/gen_stream_service.py index 04587e3532..8d8b159932 100644 --- a/scripts/gen_stream_service.py +++ b/scripts/gen_stream_service.py @@ -32,7 +32,7 @@ NEX_GEN_REPOSITORY = "https://github.com/moetemp/nexgen" # The fork chain of nexgen that reads the directives of the WIT. -NEX_GEN_REVISION = "a3842089c229131c625885aa13254699f40df501" +NEX_GEN_REVISION = "f646c68f222d4da2f31364249c5c9bb4b4b5b182" nex_gen_root = base_dir / ".nexgen-streams" / NEX_GEN_REVISION diff --git a/temporalio/api/enums/v1/__init__.py b/temporalio/api/enums/v1/__init__.py index db645f19b3..6216d437bf 100644 --- a/temporalio/api/enums/v1/__init__.py +++ b/temporalio/api/enums/v1/__init__.py @@ -45,6 +45,7 @@ from .reset_pb2 import ResetReapplyExcludeType, ResetReapplyType, ResetType from .schedule_pb2 import ScheduleOverlapPolicy from .stream_pb2 import ( + StreamCloseStatus, StreamErrorReason, StreamIdConflictPolicy, StreamReadStart, @@ -126,6 +127,7 @@ "Severity", "SignalExternalWorkflowExecutionFailedCause", "StartChildWorkflowExecutionFailedCause", + "StreamCloseStatus", "StreamErrorReason", "StreamIdConflictPolicy", "StreamReadStart", diff --git a/temporalio/api/enums/v1/stream_pb2.py b/temporalio/api/enums/v1/stream_pb2.py index b587fa6e9a..143b92ac59 100644 --- a/temporalio/api/enums/v1/stream_pb2.py +++ b/temporalio/api/enums/v1/stream_pb2.py @@ -16,11 +16,13 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n"temporal/api/enums/v1/stream.proto\x12\x15temporal.api.enums.v1*z\n\x0cStreamStatus\x12\x1d\n\x19STREAM_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12STREAM_STATUS_OPEN\x10\x01\x12\x18\n\x14STREAM_STATUS_CLOSED\x10\x02\x12\x19\n\x15STREAM_STATUS_EXPIRED\x10\x03*p\n\x0fStreamReadStart\x12!\n\x1dSTREAM_READ_START_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSTREAM_READ_START_BEGINNING\x10\x01\x12\x19\n\x15STREAM_READ_START_END\x10\x02*\x93\x01\n\x16StreamIdConflictPolicy\x12)\n%STREAM_ID_CONFLICT_POLICY_UNSPECIFIED\x10\x00\x12"\n\x1eSTREAM_ID_CONFLICT_POLICY_FAIL\x10\x01\x12*\n&STREAM_ID_CONFLICT_POLICY_USE_EXISTING\x10\x02*\x90\x05\n\x11StreamErrorReason\x12#\n\x1fSTREAM_ERROR_REASON_UNSPECIFIED\x10\x00\x12!\n\x1dSTREAM_ERROR_REASON_NOT_FOUND\x10\x01\x12\x1f\n\x1bSTREAM_ERROR_REASON_EXPIRED\x10\x02\x12\x1e\n\x1aSTREAM_ERROR_REASON_CLOSED\x10\x03\x12\x1e\n\x1aSTREAM_ERROR_REASON_FENCED\x10\x04\x12(\n$STREAM_ERROR_REASON_SEQUENCE_TOO_LOW\x10\x05\x12%\n!STREAM_ERROR_REASON_SEQ_DIVERGENT\x10\x06\x12!\n\x1dSTREAM_ERROR_REASON_NOT_OWNER\x10\x07\x12&\n"STREAM_ERROR_REASON_FORMAT_TOO_NEW\x10\x08\x12\'\n#STREAM_ERROR_REASON_OFFSET_TOO_HIGH\x10\t\x12!\n\x1dSTREAM_ERROR_REASON_TOO_LARGE\x10\n\x12\x1f\n\x1bSTREAM_ERROR_REASON_INVALID\x10\x0b\x12+\n\'STREAM_ERROR_REASON_STORAGE_UNAVAILABLE\x10\x0c\x12&\n"STREAM_ERROR_REASON_UNKNOWN_WRITER\x10\r\x12#\n\x1fSTREAM_ERROR_REASON_UNSUPPORTED\x10\x0e\x12&\n"STREAM_ERROR_REASON_ALREADY_EXISTS\x10\x0f\x12\'\n#STREAM_ERROR_REASON_STORAGE_CORRUPT\x10\x10\x42\x83\x01\n\x18io.temporal.api.enums.v1B\x0bStreamProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3' + b'\n"temporal/api/enums/v1/stream.proto\x12\x15temporal.api.enums.v1*z\n\x0cStreamStatus\x12\x1d\n\x19STREAM_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12STREAM_STATUS_OPEN\x10\x01\x12\x18\n\x14STREAM_STATUS_CLOSED\x10\x02\x12\x19\n\x15STREAM_STATUS_EXPIRED\x10\x03*\xc0\x02\n\x11StreamCloseStatus\x12#\n\x1fSTREAM_CLOSE_STATUS_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTREAM_CLOSE_STATUS_CLOSED\x10\x01\x12\'\n#STREAM_CLOSE_STATUS_OWNER_COMPLETED\x10\x02\x12$\n STREAM_CLOSE_STATUS_OWNER_FAILED\x10\x03\x12&\n"STREAM_CLOSE_STATUS_OWNER_CANCELED\x10\x04\x12(\n$STREAM_CLOSE_STATUS_OWNER_TERMINATED\x10\x05\x12\'\n#STREAM_CLOSE_STATUS_OWNER_TIMED_OUT\x10\x06\x12\x1c\n\x18STREAM_CLOSE_STATUS_IDLE\x10\x07*p\n\x0fStreamReadStart\x12!\n\x1dSTREAM_READ_START_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSTREAM_READ_START_BEGINNING\x10\x01\x12\x19\n\x15STREAM_READ_START_END\x10\x02*\x93\x01\n\x16StreamIdConflictPolicy\x12)\n%STREAM_ID_CONFLICT_POLICY_UNSPECIFIED\x10\x00\x12"\n\x1eSTREAM_ID_CONFLICT_POLICY_FAIL\x10\x01\x12*\n&STREAM_ID_CONFLICT_POLICY_USE_EXISTING\x10\x02*\x90\x05\n\x11StreamErrorReason\x12#\n\x1fSTREAM_ERROR_REASON_UNSPECIFIED\x10\x00\x12!\n\x1dSTREAM_ERROR_REASON_NOT_FOUND\x10\x01\x12\x1f\n\x1bSTREAM_ERROR_REASON_EXPIRED\x10\x02\x12\x1e\n\x1aSTREAM_ERROR_REASON_CLOSED\x10\x03\x12\x1e\n\x1aSTREAM_ERROR_REASON_FENCED\x10\x04\x12(\n$STREAM_ERROR_REASON_SEQUENCE_TOO_LOW\x10\x05\x12%\n!STREAM_ERROR_REASON_SEQ_DIVERGENT\x10\x06\x12!\n\x1dSTREAM_ERROR_REASON_NOT_OWNER\x10\x07\x12&\n"STREAM_ERROR_REASON_FORMAT_TOO_NEW\x10\x08\x12\'\n#STREAM_ERROR_REASON_OFFSET_TOO_HIGH\x10\t\x12!\n\x1dSTREAM_ERROR_REASON_TOO_LARGE\x10\n\x12\x1f\n\x1bSTREAM_ERROR_REASON_INVALID\x10\x0b\x12+\n\'STREAM_ERROR_REASON_STORAGE_UNAVAILABLE\x10\x0c\x12&\n"STREAM_ERROR_REASON_UNKNOWN_WRITER\x10\r\x12#\n\x1fSTREAM_ERROR_REASON_UNSUPPORTED\x10\x0e\x12&\n"STREAM_ERROR_REASON_ALREADY_EXISTS\x10\x0f\x12\'\n#STREAM_ERROR_REASON_STORAGE_CORRUPT\x10\x10\x42\x83\x01\n\x18io.temporal.api.enums.v1B\x0bStreamProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3' ) _STREAMSTATUS = DESCRIPTOR.enum_types_by_name["StreamStatus"] StreamStatus = enum_type_wrapper.EnumTypeWrapper(_STREAMSTATUS) +_STREAMCLOSESTATUS = DESCRIPTOR.enum_types_by_name["StreamCloseStatus"] +StreamCloseStatus = enum_type_wrapper.EnumTypeWrapper(_STREAMCLOSESTATUS) _STREAMREADSTART = DESCRIPTOR.enum_types_by_name["StreamReadStart"] StreamReadStart = enum_type_wrapper.EnumTypeWrapper(_STREAMREADSTART) _STREAMIDCONFLICTPOLICY = DESCRIPTOR.enum_types_by_name["StreamIdConflictPolicy"] @@ -31,6 +33,14 @@ STREAM_STATUS_OPEN = 1 STREAM_STATUS_CLOSED = 2 STREAM_STATUS_EXPIRED = 3 +STREAM_CLOSE_STATUS_UNSPECIFIED = 0 +STREAM_CLOSE_STATUS_CLOSED = 1 +STREAM_CLOSE_STATUS_OWNER_COMPLETED = 2 +STREAM_CLOSE_STATUS_OWNER_FAILED = 3 +STREAM_CLOSE_STATUS_OWNER_CANCELED = 4 +STREAM_CLOSE_STATUS_OWNER_TERMINATED = 5 +STREAM_CLOSE_STATUS_OWNER_TIMED_OUT = 6 +STREAM_CLOSE_STATUS_IDLE = 7 STREAM_READ_START_UNSPECIFIED = 0 STREAM_READ_START_BEGINNING = 1 STREAM_READ_START_END = 2 @@ -61,10 +71,12 @@ DESCRIPTOR._serialized_options = b"\n\030io.temporal.api.enums.v1B\013StreamProtoP\001Z!go.temporal.io/api/enums/v1;enums\252\002\027Temporalio.Api.Enums.V1\352\002\032Temporalio::Api::Enums::V1" _STREAMSTATUS._serialized_start = 61 _STREAMSTATUS._serialized_end = 183 - _STREAMREADSTART._serialized_start = 185 - _STREAMREADSTART._serialized_end = 297 - _STREAMIDCONFLICTPOLICY._serialized_start = 300 - _STREAMIDCONFLICTPOLICY._serialized_end = 447 - _STREAMERRORREASON._serialized_start = 450 - _STREAMERRORREASON._serialized_end = 1106 + _STREAMCLOSESTATUS._serialized_start = 186 + _STREAMCLOSESTATUS._serialized_end = 506 + _STREAMREADSTART._serialized_start = 508 + _STREAMREADSTART._serialized_end = 620 + _STREAMIDCONFLICTPOLICY._serialized_start = 623 + _STREAMIDCONFLICTPOLICY._serialized_end = 770 + _STREAMERRORREASON._serialized_start = 773 + _STREAMERRORREASON._serialized_end = 1429 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/enums/v1/stream_pb2.pyi b/temporalio/api/enums/v1/stream_pb2.pyi index e314317c76..2d026f71c8 100644 --- a/temporalio/api/enums/v1/stream_pb2.pyi +++ b/temporalio/api/enums/v1/stream_pb2.pyi @@ -46,6 +46,47 @@ STREAM_STATUS_CLOSED: StreamStatus.ValueType # 2 STREAM_STATUS_EXPIRED: StreamStatus.ValueType # 3 global___StreamStatus = StreamStatus +class _StreamCloseStatus: + ValueType = typing.NewType("ValueType", builtins.int) + V: typing_extensions.TypeAlias = ValueType + +class _StreamCloseStatusEnumTypeWrapper( + google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[ + _StreamCloseStatus.ValueType + ], + builtins.type, +): # noqa: F821 + DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor + STREAM_CLOSE_STATUS_UNSPECIFIED: _StreamCloseStatus.ValueType # 0 + STREAM_CLOSE_STATUS_CLOSED: _StreamCloseStatus.ValueType # 1 + STREAM_CLOSE_STATUS_OWNER_COMPLETED: _StreamCloseStatus.ValueType # 2 + STREAM_CLOSE_STATUS_OWNER_FAILED: _StreamCloseStatus.ValueType # 3 + STREAM_CLOSE_STATUS_OWNER_CANCELED: _StreamCloseStatus.ValueType # 4 + STREAM_CLOSE_STATUS_OWNER_TERMINATED: _StreamCloseStatus.ValueType # 5 + STREAM_CLOSE_STATUS_OWNER_TIMED_OUT: _StreamCloseStatus.ValueType # 6 + STREAM_CLOSE_STATUS_IDLE: _StreamCloseStatus.ValueType # 7 + +class StreamCloseStatus( + _StreamCloseStatus, metaclass=_StreamCloseStatusEnumTypeWrapper +): + """Experimental. Might change in the future. + How a Stream closed. CLOSED comes from a Producer that called close(). The OWNER_ values and + IDLE come from the server (native Streams) when the owner ends or the Stream goes idle. A Brook + SDK store only emits CLOSED. + (-- api-linter: core::0216::synonyms=disabled + aip.dev/not-precedent: Named consistently with WorkflowExecutionStatus. --) + """ + +STREAM_CLOSE_STATUS_UNSPECIFIED: StreamCloseStatus.ValueType # 0 +STREAM_CLOSE_STATUS_CLOSED: StreamCloseStatus.ValueType # 1 +STREAM_CLOSE_STATUS_OWNER_COMPLETED: StreamCloseStatus.ValueType # 2 +STREAM_CLOSE_STATUS_OWNER_FAILED: StreamCloseStatus.ValueType # 3 +STREAM_CLOSE_STATUS_OWNER_CANCELED: StreamCloseStatus.ValueType # 4 +STREAM_CLOSE_STATUS_OWNER_TERMINATED: StreamCloseStatus.ValueType # 5 +STREAM_CLOSE_STATUS_OWNER_TIMED_OUT: StreamCloseStatus.ValueType # 6 +STREAM_CLOSE_STATUS_IDLE: StreamCloseStatus.ValueType # 7 +global___StreamCloseStatus = StreamCloseStatus + class _StreamReadStart: ValueType = typing.NewType("ValueType", builtins.int) V: typing_extensions.TypeAlias = ValueType diff --git a/temporalio/api/stream/v1/__init__.py b/temporalio/api/stream/v1/__init__.py index 6232b08cc0..fdd4920124 100644 --- a/temporalio/api/stream/v1/__init__.py +++ b/temporalio/api/stream/v1/__init__.py @@ -1,31 +1,37 @@ from .message_pb2 import ( ActivityEpoch, + ActivityProducer, AppendRecord, ClientEpoch, + ClientProducer, LocalActivityEpoch, + LocalActivityProducer, Producer, - ProducerEpoch, + StreamClose, StreamData, StreamErrorDetails, StreamRecord, StreamRef, - StreamSuperseded, StreamWriter, + WorkflowProducer, WorkflowTaskEpoch, ) __all__ = [ "ActivityEpoch", + "ActivityProducer", "AppendRecord", "ClientEpoch", + "ClientProducer", "LocalActivityEpoch", + "LocalActivityProducer", "Producer", - "ProducerEpoch", + "StreamClose", "StreamData", "StreamErrorDetails", "StreamRecord", "StreamRef", - "StreamSuperseded", "StreamWriter", + "WorkflowProducer", "WorkflowTaskEpoch", ] diff --git a/temporalio/api/stream/v1/message_pb2.py b/temporalio/api/stream/v1/message_pb2.py index c3f561c549..846e89a5ac 100644 --- a/temporalio/api/stream/v1/message_pb2.py +++ b/temporalio/api/stream/v1/message_pb2.py @@ -22,23 +22,29 @@ from temporalio.api.enums.v1 import ( stream_pb2 as temporal_dot_api_dot_enums_dot_v1_dot_stream__pb2, ) +from temporalio.api.failure.v1 import ( + message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, +) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n$temporal/api/stream/v1/message.proto\x12\x16temporal.api.stream.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a"temporal/api/enums/v1/stream.proto"Z\n\tStreamRef\x12\n\n\x02id\x18\x01 \x01(\t\x12\x30\n\x05owner\x18\x02 \x01(\x0b\x32!.temporal.api.common.v1.Execution\x12\x0f\n\x07\x62\x61\x63kend\x18\x03 \x01(\t"\xd4\x02\n\rProducerEpoch\x12=\n\x08workflow\x18\x01 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpochH\x00\x12\x44\n\x0elocal_activity\x18\x02 \x01(\x0b\x32*.temporal.api.stream.v1.LocalActivityEpochH\x00\x12\x39\n\x08\x61\x63tivity\x18\x03 \x01(\x0b\x32%.temporal.api.stream.v1.ActivityEpochH\x00\x12\x44\n\x13standalone_activity\x18\x04 \x01(\x0b\x32%.temporal.api.stream.v1.ActivityEpochH\x00\x12\x35\n\x06\x63lient\x18\x05 \x01(\x0b\x32#.temporal.api.stream.v1.ClientEpochH\x00\x42\x06\n\x04kind"p\n\x11WorkflowTaskEpoch\x12\x18\n\x10started_event_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05\x12\x30\n\x0cstarted_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"^\n\x12LocalActivityEpoch\x12\x37\n\x04task\x18\x01 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05"\\\n\rActivityEpoch\x12:\n\x16\x61ttempt_scheduled_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05"\x1e\n\x0b\x43lientEpoch\x12\x0f\n\x07\x61ttempt\x18\x01 \x01(\x05"\\\n\x08Producer\x12\n\n\x02id\x18\x01 \x01(\t\x12\x34\n\x05\x65poch\x18\x02 \x01(\x0b\x32%.temporal.api.stream.v1.ProducerEpoch\x12\x0e\n\x06run_id\x18\x03 \x01(\t"\xba\x01\n\x0cStreamWriter\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12J\n\x17previous_completed_task\x18\x02 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x14\n\x0c\x66\x61iled_tasks\x18\x03 \x03(\x03\x12\x14\n\x0cwithdraw_run\x18\x04 \x01(\t"=\n\x0c\x41ppendRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"H\n\x10StreamSuperseded\x12\x34\n\x06writer\x18\x01 \x01(\x0b\x32$.temporal.api.stream.v1.StreamWriter"o\n\nStreamData\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12-\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"\x9a\x01\n\x0cStreamRecord\x12\x0e\n\x06offset\x18\x01 \x01(\x03\x12\x32\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32".temporal.api.stream.v1.StreamDataH\x00\x12>\n\nsuperseded\x18\x03 \x01(\x0b\x32(.temporal.api.stream.v1.StreamSupersededH\x00\x42\x06\n\x04item"N\n\x12StreamErrorDetails\x12\x38\n\x06reason\x18\x01 \x01(\x0e\x32(.temporal.api.enums.v1.StreamErrorReasonB\x89\x01\n\x19io.temporal.api.stream.v1B\x0cMessageProtoP\x01Z#go.temporal.io/api/stream/v1;stream\xaa\x02\x18Temporalio.Api.Stream.V1\xea\x02\x1bTemporalio::Api::Stream::V1b\x06proto3' + b'\n$temporal/api/stream/v1/message.proto\x12\x16temporal.api.stream.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a"temporal/api/enums/v1/stream.proto\x1a%temporal/api/failure/v1/message.proto"(\n\tStreamRef\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07\x62\x61\x63kend\x18\x03 \x01(\t"p\n\x11WorkflowTaskEpoch\x12\x18\n\x10started_event_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05\x12\x30\n\x0cstarted_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"^\n\x12LocalActivityEpoch\x12\x37\n\x04task\x18\x01 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05"\\\n\rActivityEpoch\x12:\n\x16\x61ttempt_scheduled_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05"\x1e\n\x0b\x43lientEpoch\x12\x0f\n\x07\x61ttempt\x18\x01 \x01(\x05"\\\n\x10WorkflowProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x38\n\x05\x65poch\x18\x02 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch"b\n\x15LocalActivityProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x39\n\x05\x65poch\x18\x02 \x01(\x0b\x32*.temporal.api.stream.v1.LocalActivityEpoch"X\n\x10\x41\x63tivityProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x34\n\x05\x65poch\x18\x02 \x01(\x0b\x32%.temporal.api.stream.v1.ActivityEpoch"D\n\x0e\x43lientProducer\x12\x32\n\x05\x65poch\x18\x01 \x01(\x0b\x32#.temporal.api.stream.v1.ClientEpoch"\xe6\x02\n\x08Producer\x12\n\n\x02id\x18\x01 \x01(\t\x12<\n\x08workflow\x18\x02 \x01(\x0b\x32(.temporal.api.stream.v1.WorkflowProducerH\x00\x12G\n\x0elocal_activity\x18\x03 \x01(\x0b\x32-.temporal.api.stream.v1.LocalActivityProducerH\x00\x12<\n\x08\x61\x63tivity\x18\x04 \x01(\x0b\x32(.temporal.api.stream.v1.ActivityProducerH\x00\x12G\n\x13standalone_activity\x18\x05 \x01(\x0b\x32(.temporal.api.stream.v1.ActivityProducerH\x00\x12\x38\n\x06\x63lient\x18\x06 \x01(\x0b\x32&.temporal.api.stream.v1.ClientProducerH\x00\x42\x06\n\x04kind"B\n\x0cStreamWriter\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer"=\n\x0c\x41ppendRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"o\n\nStreamData\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12-\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"\xdf\x01\n\x0bStreamClose\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12/\n\x06result\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x31\n\x07\x66\x61ilure\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x38\n\x06status\x18\x04 \x01(\x0e\x32(.temporal.api.enums.v1.StreamCloseStatus"\x90\x01\n\x0cStreamRecord\x12\x0e\n\x06offset\x18\x01 \x01(\x03\x12\x32\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32".temporal.api.stream.v1.StreamDataH\x00\x12\x34\n\x05\x63lose\x18\x04 \x01(\x0b\x32#.temporal.api.stream.v1.StreamCloseH\x00\x42\x06\n\x04item"N\n\x12StreamErrorDetails\x12\x38\n\x06reason\x18\x01 \x01(\x0e\x32(.temporal.api.enums.v1.StreamErrorReasonB\x89\x01\n\x19io.temporal.api.stream.v1B\x0cMessageProtoP\x01Z#go.temporal.io/api/stream/v1;stream\xaa\x02\x18Temporalio.Api.Stream.V1\xea\x02\x1bTemporalio::Api::Stream::V1b\x06proto3' ) _STREAMREF = DESCRIPTOR.message_types_by_name["StreamRef"] -_PRODUCEREPOCH = DESCRIPTOR.message_types_by_name["ProducerEpoch"] _WORKFLOWTASKEPOCH = DESCRIPTOR.message_types_by_name["WorkflowTaskEpoch"] _LOCALACTIVITYEPOCH = DESCRIPTOR.message_types_by_name["LocalActivityEpoch"] _ACTIVITYEPOCH = DESCRIPTOR.message_types_by_name["ActivityEpoch"] _CLIENTEPOCH = DESCRIPTOR.message_types_by_name["ClientEpoch"] +_WORKFLOWPRODUCER = DESCRIPTOR.message_types_by_name["WorkflowProducer"] +_LOCALACTIVITYPRODUCER = DESCRIPTOR.message_types_by_name["LocalActivityProducer"] +_ACTIVITYPRODUCER = DESCRIPTOR.message_types_by_name["ActivityProducer"] +_CLIENTPRODUCER = DESCRIPTOR.message_types_by_name["ClientProducer"] _PRODUCER = DESCRIPTOR.message_types_by_name["Producer"] _STREAMWRITER = DESCRIPTOR.message_types_by_name["StreamWriter"] _APPENDRECORD = DESCRIPTOR.message_types_by_name["AppendRecord"] -_STREAMSUPERSEDED = DESCRIPTOR.message_types_by_name["StreamSuperseded"] _STREAMDATA = DESCRIPTOR.message_types_by_name["StreamData"] +_STREAMCLOSE = DESCRIPTOR.message_types_by_name["StreamClose"] _STREAMRECORD = DESCRIPTOR.message_types_by_name["StreamRecord"] _STREAMERRORDETAILS = DESCRIPTOR.message_types_by_name["StreamErrorDetails"] StreamRef = _reflection.GeneratedProtocolMessageType( @@ -52,17 +58,6 @@ ) _sym_db.RegisterMessage(StreamRef) -ProducerEpoch = _reflection.GeneratedProtocolMessageType( - "ProducerEpoch", - (_message.Message,), - { - "DESCRIPTOR": _PRODUCEREPOCH, - "__module__": "temporalio.api.stream.v1.message_pb2", - # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.ProducerEpoch) - }, -) -_sym_db.RegisterMessage(ProducerEpoch) - WorkflowTaskEpoch = _reflection.GeneratedProtocolMessageType( "WorkflowTaskEpoch", (_message.Message,), @@ -107,6 +102,50 @@ ) _sym_db.RegisterMessage(ClientEpoch) +WorkflowProducer = _reflection.GeneratedProtocolMessageType( + "WorkflowProducer", + (_message.Message,), + { + "DESCRIPTOR": _WORKFLOWPRODUCER, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.WorkflowProducer) + }, +) +_sym_db.RegisterMessage(WorkflowProducer) + +LocalActivityProducer = _reflection.GeneratedProtocolMessageType( + "LocalActivityProducer", + (_message.Message,), + { + "DESCRIPTOR": _LOCALACTIVITYPRODUCER, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.LocalActivityProducer) + }, +) +_sym_db.RegisterMessage(LocalActivityProducer) + +ActivityProducer = _reflection.GeneratedProtocolMessageType( + "ActivityProducer", + (_message.Message,), + { + "DESCRIPTOR": _ACTIVITYPRODUCER, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.ActivityProducer) + }, +) +_sym_db.RegisterMessage(ActivityProducer) + +ClientProducer = _reflection.GeneratedProtocolMessageType( + "ClientProducer", + (_message.Message,), + { + "DESCRIPTOR": _CLIENTPRODUCER, + "__module__": "temporalio.api.stream.v1.message_pb2", + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.ClientProducer) + }, +) +_sym_db.RegisterMessage(ClientProducer) + Producer = _reflection.GeneratedProtocolMessageType( "Producer", (_message.Message,), @@ -140,27 +179,27 @@ ) _sym_db.RegisterMessage(AppendRecord) -StreamSuperseded = _reflection.GeneratedProtocolMessageType( - "StreamSuperseded", +StreamData = _reflection.GeneratedProtocolMessageType( + "StreamData", (_message.Message,), { - "DESCRIPTOR": _STREAMSUPERSEDED, + "DESCRIPTOR": _STREAMDATA, "__module__": "temporalio.api.stream.v1.message_pb2", - # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamSuperseded) + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamData) }, ) -_sym_db.RegisterMessage(StreamSuperseded) +_sym_db.RegisterMessage(StreamData) -StreamData = _reflection.GeneratedProtocolMessageType( - "StreamData", +StreamClose = _reflection.GeneratedProtocolMessageType( + "StreamClose", (_message.Message,), { - "DESCRIPTOR": _STREAMDATA, + "DESCRIPTOR": _STREAMCLOSE, "__module__": "temporalio.api.stream.v1.message_pb2", - # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamData) + # @@protoc_insertion_point(class_scope:temporal.api.stream.v1.StreamClose) }, ) -_sym_db.RegisterMessage(StreamData) +_sym_db.RegisterMessage(StreamClose) StreamRecord = _reflection.GeneratedProtocolMessageType( "StreamRecord", @@ -187,30 +226,36 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\031io.temporal.api.stream.v1B\014MessageProtoP\001Z#go.temporal.io/api/stream/v1;stream\252\002\030Temporalio.Api.Stream.V1\352\002\033Temporalio::Api::Stream::V1" - _STREAMREF._serialized_start = 171 - _STREAMREF._serialized_end = 261 - _PRODUCEREPOCH._serialized_start = 264 - _PRODUCEREPOCH._serialized_end = 604 - _WORKFLOWTASKEPOCH._serialized_start = 606 - _WORKFLOWTASKEPOCH._serialized_end = 718 - _LOCALACTIVITYEPOCH._serialized_start = 720 - _LOCALACTIVITYEPOCH._serialized_end = 814 - _ACTIVITYEPOCH._serialized_start = 816 - _ACTIVITYEPOCH._serialized_end = 908 - _CLIENTEPOCH._serialized_start = 910 - _CLIENTEPOCH._serialized_end = 940 - _PRODUCER._serialized_start = 942 - _PRODUCER._serialized_end = 1034 - _STREAMWRITER._serialized_start = 1037 - _STREAMWRITER._serialized_end = 1223 - _APPENDRECORD._serialized_start = 1225 - _APPENDRECORD._serialized_end = 1286 - _STREAMSUPERSEDED._serialized_start = 1288 - _STREAMSUPERSEDED._serialized_end = 1360 - _STREAMDATA._serialized_start = 1362 - _STREAMDATA._serialized_end = 1473 - _STREAMRECORD._serialized_start = 1476 - _STREAMRECORD._serialized_end = 1630 - _STREAMERRORDETAILS._serialized_start = 1632 - _STREAMERRORDETAILS._serialized_end = 1710 + _STREAMREF._serialized_start = 210 + _STREAMREF._serialized_end = 250 + _WORKFLOWTASKEPOCH._serialized_start = 252 + _WORKFLOWTASKEPOCH._serialized_end = 364 + _LOCALACTIVITYEPOCH._serialized_start = 366 + _LOCALACTIVITYEPOCH._serialized_end = 460 + _ACTIVITYEPOCH._serialized_start = 462 + _ACTIVITYEPOCH._serialized_end = 554 + _CLIENTEPOCH._serialized_start = 556 + _CLIENTEPOCH._serialized_end = 586 + _WORKFLOWPRODUCER._serialized_start = 588 + _WORKFLOWPRODUCER._serialized_end = 680 + _LOCALACTIVITYPRODUCER._serialized_start = 682 + _LOCALACTIVITYPRODUCER._serialized_end = 780 + _ACTIVITYPRODUCER._serialized_start = 782 + _ACTIVITYPRODUCER._serialized_end = 870 + _CLIENTPRODUCER._serialized_start = 872 + _CLIENTPRODUCER._serialized_end = 940 + _PRODUCER._serialized_start = 943 + _PRODUCER._serialized_end = 1301 + _STREAMWRITER._serialized_start = 1303 + _STREAMWRITER._serialized_end = 1369 + _APPENDRECORD._serialized_start = 1371 + _APPENDRECORD._serialized_end = 1432 + _STREAMDATA._serialized_start = 1434 + _STREAMDATA._serialized_end = 1545 + _STREAMCLOSE._serialized_start = 1548 + _STREAMCLOSE._serialized_end = 1771 + _STREAMRECORD._serialized_start = 1774 + _STREAMRECORD._serialized_end = 1918 + _STREAMERRORDETAILS._serialized_start = 1920 + _STREAMERRORDETAILS._serialized_end = 1998 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/stream/v1/message_pb2.pyi b/temporalio/api/stream/v1/message_pb2.pyi index a8dc53d3ec..277bff996c 100644 --- a/temporalio/api/stream/v1/message_pb2.pyi +++ b/temporalio/api/stream/v1/message_pb2.pyi @@ -4,16 +4,15 @@ isort:skip_file """ import builtins -import collections.abc import sys import google.protobuf.descriptor -import google.protobuf.internal.containers import google.protobuf.message import google.protobuf.timestamp_pb2 import temporalio.api.common.v1.message_pb2 import temporalio.api.enums.v1.stream_pb2 +import temporalio.api.failure.v1.message_pb2 if sys.version_info >= (3, 8): import typing as typing_extensions @@ -30,16 +29,8 @@ class StreamRef(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor ID_FIELD_NUMBER: builtins.int - OWNER_FIELD_NUMBER: builtins.int BACKEND_FIELD_NUMBER: builtins.int id: builtins.str - @property - def owner(self) -> temporalio.api.common.v1.message_pb2.Execution: - """Unset for a standalone Stream. For a Stream bound to a Workflow, business_id is the Workflow - ID and run_id is the first execution run id of its run chain. For a Standalone Activity, - business_id is the activity ID and run_id is its run id. A create and a close of a - Workflow-owned Stream must carry its owner. - """ backend: builtins.str """The name of the backend that holds the Stream, as the Worker and client config register it, so every process routes a ref it receives. Empty is the default backend. The Store @@ -49,108 +40,14 @@ class StreamRef(google.protobuf.message.Message): self, *, id: builtins.str = ..., - owner: temporalio.api.common.v1.message_pb2.Execution | None = ..., backend: builtins.str = ..., ) -> None: ... - def HasField( - self, field_name: typing_extensions.Literal["owner", b"owner"] - ) -> builtins.bool: ... def ClearField( - self, - field_name: typing_extensions.Literal[ - "backend", b"backend", "id", b"id", "owner", b"owner" - ], + self, field_name: typing_extensions.Literal["backend", b"backend", "id", b"id"] ) -> None: ... global___StreamRef = StreamRef -class ProducerEpoch(google.protobuf.message.Message): - """Experimental. Might change in the future. - The ordered attempt of a Producer. A higher epoch supersedes the Records of a lower epoch of the - same Producer. The case is the kind of the Producer. It decides how the Store orders the epochs, - and which earlier Records a higher epoch withdraws. - """ - - DESCRIPTOR: google.protobuf.descriptor.Descriptor - - WORKFLOW_FIELD_NUMBER: builtins.int - LOCAL_ACTIVITY_FIELD_NUMBER: builtins.int - ACTIVITY_FIELD_NUMBER: builtins.int - STANDALONE_ACTIVITY_FIELD_NUMBER: builtins.int - CLIENT_FIELD_NUMBER: builtins.int - @property - def workflow(self) -> global___WorkflowTaskEpoch: - """A Workflow run, which writes in its Workflow Task. A higher epoch withdraws the Records - of the run's Workflow Tasks that never completed, also those of their Local Activities. - """ - @property - def local_activity(self) -> global___LocalActivityEpoch: - """A Local Activity. A higher epoch withdraws the Producer's lower epochs.""" - @property - def activity(self) -> global___ActivityEpoch: - """An Activity of a Workflow. A higher epoch withdraws the Producer's lower epochs.""" - @property - def standalone_activity(self) -> global___ActivityEpoch: - """A Standalone Activity. A higher epoch withdraws the Producer's lower epochs.""" - @property - def client(self) -> global___ClientEpoch: - """A client. A higher epoch only fences the lower ones, since a restarted client's earlier - Records are real messages, not the Records of a failed attempt. - """ - def __init__( - self, - *, - workflow: global___WorkflowTaskEpoch | None = ..., - local_activity: global___LocalActivityEpoch | None = ..., - activity: global___ActivityEpoch | None = ..., - standalone_activity: global___ActivityEpoch | None = ..., - client: global___ClientEpoch | None = ..., - ) -> None: ... - def HasField( - self, - field_name: typing_extensions.Literal[ - "activity", - b"activity", - "client", - b"client", - "kind", - b"kind", - "local_activity", - b"local_activity", - "standalone_activity", - b"standalone_activity", - "workflow", - b"workflow", - ], - ) -> builtins.bool: ... - def ClearField( - self, - field_name: typing_extensions.Literal[ - "activity", - b"activity", - "client", - b"client", - "kind", - b"kind", - "local_activity", - b"local_activity", - "standalone_activity", - b"standalone_activity", - "workflow", - b"workflow", - ], - ) -> None: ... - def WhichOneof( - self, oneof_group: typing_extensions.Literal["kind", b"kind"] - ) -> ( - typing_extensions.Literal[ - "workflow", "local_activity", "activity", "standalone_activity", "client" - ] - | None - ): ... - -global___ProducerEpoch = ProducerEpoch - class WorkflowTaskEpoch(google.protobuf.message.Message): """Experimental. Might change in the future. One Workflow Task attempt of a run. It orders by started_event_id, then attempt, then @@ -278,159 +175,251 @@ class ClientEpoch(google.protobuf.message.Message): global___ClientEpoch = ClientEpoch -class Producer(google.protobuf.message.Message): +class WorkflowProducer(google.protobuf.message.Message): """Experimental. Might change in the future. - The code that wrote a batch. Its id stays the same across attempts, and its epoch orders them. + A Workflow run Producer. It writes in its Workflow Task. A higher epoch withdraws the Records of + the run's Workflow Tasks that never completed, with those of their Local Activities. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor - ID_FIELD_NUMBER: builtins.int - EPOCH_FIELD_NUMBER: builtins.int RUN_ID_FIELD_NUMBER: builtins.int - id: builtins.str - """Stable across attempts, such as `wf:` for the Workflow Producer of a run.""" - @property - def epoch(self) -> global___ProducerEpoch: - """Required. Its case gives the kind of the Producer.""" + EPOCH_FIELD_NUMBER: builtins.int run_id: builtins.str - """The run of the Producer, so nobody parses the id. Empty for a client.""" + """The first execution run id of the run chain.""" + @property + def epoch(self) -> global___WorkflowTaskEpoch: ... def __init__( self, *, - id: builtins.str = ..., - epoch: global___ProducerEpoch | None = ..., run_id: builtins.str = ..., + epoch: global___WorkflowTaskEpoch | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal["epoch", b"epoch"] ) -> builtins.bool: ... def ClearField( self, - field_name: typing_extensions.Literal[ - "epoch", b"epoch", "id", b"id", "run_id", b"run_id" - ], + field_name: typing_extensions.Literal["epoch", b"epoch", "run_id", b"run_id"], ) -> None: ... -global___Producer = Producer +global___WorkflowProducer = WorkflowProducer -class StreamWriter(google.protobuf.message.Message): +class LocalActivityProducer(google.protobuf.message.Message): """Experimental. Might change in the future. - The Producer of a write, and which earlier Records the write withdraws. An append, a close and - a writer carry the same fields, so the Store orders and withdraws each write the same way. + A Local Activity Producer. A higher epoch withdraws the Producer's lower epochs. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor - PRODUCER_FIELD_NUMBER: builtins.int - PREVIOUS_COMPLETED_TASK_FIELD_NUMBER: builtins.int - FAILED_TASKS_FIELD_NUMBER: builtins.int - WITHDRAW_RUN_FIELD_NUMBER: builtins.int + RUN_ID_FIELD_NUMBER: builtins.int + EPOCH_FIELD_NUMBER: builtins.int + run_id: builtins.str @property - def producer(self) -> global___Producer: - """Required on a write. Its epoch's case picks the withdrawal rule.""" + def epoch(self) -> global___LocalActivityEpoch: ... + def __init__( + self, + *, + run_id: builtins.str = ..., + epoch: global___LocalActivityEpoch | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["epoch", b"epoch"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["epoch", b"epoch", "run_id", b"run_id"], + ) -> None: ... + +global___LocalActivityProducer = LocalActivityProducer + +class ActivityProducer(google.protobuf.message.Message): + """Experimental. Might change in the future. + An Activity Producer, for an Activity of a Workflow or a Standalone Activity. A higher epoch + withdraws the Producer's lower epochs. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + RUN_ID_FIELD_NUMBER: builtins.int + EPOCH_FIELD_NUMBER: builtins.int + run_id: builtins.str @property - def previous_completed_task(self) -> global___WorkflowTaskEpoch: - """Workflow and Local Activity only. The run's last completed Workflow Task, from a Local - Activity's start. Its started_event_id is 0 before the first. The write withdraws the run's - Local Activity Records whose Workflow Task never completed. - """ + def epoch(self) -> global___ActivityEpoch: ... + def __init__( + self, + *, + run_id: builtins.str = ..., + epoch: global___ActivityEpoch | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["epoch", b"epoch"] + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal["epoch", b"epoch", "run_id", b"run_id"], + ) -> None: ... + +global___ActivityProducer = ActivityProducer + +class ClientProducer(google.protobuf.message.Message): + """Experimental. Might change in the future. + A client Producer. A higher epoch only fences the lower ones, since a restarted client's earlier + Records are real messages, not the Records of a failed attempt. A client has no run. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + EPOCH_FIELD_NUMBER: builtins.int @property - def failed_tasks( + def epoch(self) -> global___ClientEpoch: ... + def __init__( self, - ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: - """Workflow and Local Activity only. The started event ids of the run's failed or timed out - Workflow Tasks. The write withdraws the run's Local Activity Records of these tasks. - """ - withdraw_run: builtins.str - """Retry run only. The previous run, on the retry run's first write. Every Record whose - Producer carries it is withdrawn. + *, + epoch: global___ClientEpoch | None = ..., + ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["epoch", b"epoch"] + ) -> builtins.bool: ... + def ClearField( + self, field_name: typing_extensions.Literal["epoch", b"epoch"] + ) -> None: ... + +global___ClientProducer = ClientProducer + +class Producer(google.protobuf.message.Message): + """Experimental. Might change in the future. + The code that wrote a batch. Its id stays the same across attempts, and the kind's case orders + its epochs and picks which earlier Records a higher epoch withdraws. A run-based kind carries its + run, so a client can't. """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ID_FIELD_NUMBER: builtins.int + WORKFLOW_FIELD_NUMBER: builtins.int + LOCAL_ACTIVITY_FIELD_NUMBER: builtins.int + ACTIVITY_FIELD_NUMBER: builtins.int + STANDALONE_ACTIVITY_FIELD_NUMBER: builtins.int + CLIENT_FIELD_NUMBER: builtins.int + id: builtins.str + """Stable across attempts, such as `wf:` for the Workflow Producer of a run.""" + @property + def workflow(self) -> global___WorkflowProducer: ... + @property + def local_activity(self) -> global___LocalActivityProducer: ... + @property + def activity(self) -> global___ActivityProducer: ... + @property + def standalone_activity(self) -> global___ActivityProducer: ... + @property + def client(self) -> global___ClientProducer: ... def __init__( self, *, - producer: global___Producer | None = ..., - previous_completed_task: global___WorkflowTaskEpoch | None = ..., - failed_tasks: collections.abc.Iterable[builtins.int] | None = ..., - withdraw_run: builtins.str = ..., + id: builtins.str = ..., + workflow: global___WorkflowProducer | None = ..., + local_activity: global___LocalActivityProducer | None = ..., + activity: global___ActivityProducer | None = ..., + standalone_activity: global___ActivityProducer | None = ..., + client: global___ClientProducer | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ - "previous_completed_task", - b"previous_completed_task", - "producer", - b"producer", + "activity", + b"activity", + "client", + b"client", + "kind", + b"kind", + "local_activity", + b"local_activity", + "standalone_activity", + b"standalone_activity", + "workflow", + b"workflow", ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ - "failed_tasks", - b"failed_tasks", - "previous_completed_task", - b"previous_completed_task", - "producer", - b"producer", - "withdraw_run", - b"withdraw_run", + "activity", + b"activity", + "client", + b"client", + "id", + b"id", + "kind", + b"kind", + "local_activity", + b"local_activity", + "standalone_activity", + b"standalone_activity", + "workflow", + b"workflow", ], ) -> None: ... + def WhichOneof( + self, oneof_group: typing_extensions.Literal["kind", b"kind"] + ) -> ( + typing_extensions.Literal[ + "workflow", "local_activity", "activity", "standalone_activity", "client" + ] + | None + ): ... -global___StreamWriter = StreamWriter +global___Producer = Producer -class AppendRecord(google.protobuf.message.Message): +class StreamWriter(google.protobuf.message.Message): """Experimental. Might change in the future. - One Record of a batch to append. The Store adds the batch's Producer and an offset. + The Producer of a write. An append and a close carry the same writer, so the Store orders each + write the same way. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor - BODY_FIELD_NUMBER: builtins.int + PRODUCER_FIELD_NUMBER: builtins.int @property - def body(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + def producer(self) -> global___Producer: + """Required on a write.""" def __init__( self, *, - body: temporalio.api.common.v1.message_pb2.Payload | None = ..., + producer: global___Producer | None = ..., ) -> None: ... def HasField( - self, field_name: typing_extensions.Literal["body", b"body"] + self, field_name: typing_extensions.Literal["producer", b"producer"] ) -> builtins.bool: ... def ClearField( - self, field_name: typing_extensions.Literal["body", b"body"] + self, field_name: typing_extensions.Literal["producer", b"producer"] ) -> None: ... -global___AppendRecord = AppendRecord +global___StreamWriter = StreamWriter -class StreamSuperseded(google.protobuf.message.Message): +class AppendRecord(google.protobuf.message.Message): """Experimental. Might change in the future. - Which earlier Records a retried Producer withdraws. The Store keeps these facts for the first - batch of a higher epoch, or for a retry run's first write, and a read builds a Superseded item - from them. A reader drops the earlier Records that the facts match. + One Record of a batch to append. The Store adds the batch's Producer and an offset. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor - WRITER_FIELD_NUMBER: builtins.int + BODY_FIELD_NUMBER: builtins.int @property - def writer(self) -> global___StreamWriter: - """The writer of the batch that withdraws. Its Producer's epoch is empty when only - withdraw_run applies. - """ + def body(self) -> temporalio.api.common.v1.message_pb2.Payload: ... def __init__( self, *, - writer: global___StreamWriter | None = ..., + body: temporalio.api.common.v1.message_pb2.Payload | None = ..., ) -> None: ... def HasField( - self, field_name: typing_extensions.Literal["writer", b"writer"] + self, field_name: typing_extensions.Literal["body", b"body"] ) -> builtins.bool: ... def ClearField( - self, field_name: typing_extensions.Literal["writer", b"writer"] + self, field_name: typing_extensions.Literal["body", b"body"] ) -> None: ... -global___StreamSuperseded = StreamSuperseded +global___AppendRecord = AppendRecord class StreamData(google.protobuf.message.Message): """Experimental. Might change in the future. @@ -462,6 +451,56 @@ class StreamData(google.protobuf.message.Message): global___StreamData = StreamData +class StreamClose(google.protobuf.message.Message): + """Experimental. Might change in the future. + The close of a Stream. A Stream has at most one, and a read returns it last. + """ + + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + PRODUCER_FIELD_NUMBER: builtins.int + RESULT_FIELD_NUMBER: builtins.int + FAILURE_FIELD_NUMBER: builtins.int + STATUS_FIELD_NUMBER: builtins.int + @property + def producer(self) -> global___Producer: + """The Producer that closed the Stream.""" + @property + def result(self) -> temporalio.api.common.v1.message_pb2.Payload: ... + @property + def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... + status: temporalio.api.enums.v1.stream_pb2.StreamCloseStatus.ValueType + """Which of result or failure is set, and why the Stream closed.""" + def __init__( + self, + *, + producer: global___Producer | None = ..., + result: temporalio.api.common.v1.message_pb2.Payload | None = ..., + failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., + status: temporalio.api.enums.v1.stream_pb2.StreamCloseStatus.ValueType = ..., + ) -> None: ... + def HasField( + self, + field_name: typing_extensions.Literal[ + "failure", b"failure", "producer", b"producer", "result", b"result" + ], + ) -> builtins.bool: ... + def ClearField( + self, + field_name: typing_extensions.Literal[ + "failure", + b"failure", + "producer", + b"producer", + "result", + b"result", + "status", + b"status", + ], + ) -> None: ... + +global___StreamClose = StreamClose + class StreamRecord(google.protobuf.message.Message): """Experimental. Might change in the future. An offset is ordered and unique in one Stream, and it is sparse. Offsets have gaps, so they @@ -473,45 +512,34 @@ class StreamRecord(google.protobuf.message.Message): OFFSET_FIELD_NUMBER: builtins.int DATA_FIELD_NUMBER: builtins.int - SUPERSEDED_FIELD_NUMBER: builtins.int + CLOSE_FIELD_NUMBER: builtins.int offset: builtins.int @property def data(self) -> global___StreamData: ... @property - def superseded(self) -> global___StreamSuperseded: - """The Store keeps only the withdrawal facts of the first batch of a higher epoch. A read - builds this item from them. It withdraws the earlier Records that the facts match, and a - reader drops them. - """ + def close(self) -> global___StreamClose: ... def __init__( self, *, offset: builtins.int = ..., data: global___StreamData | None = ..., - superseded: global___StreamSuperseded | None = ..., + close: global___StreamClose | None = ..., ) -> None: ... def HasField( self, field_name: typing_extensions.Literal[ - "data", b"data", "item", b"item", "superseded", b"superseded" + "close", b"close", "data", b"data", "item", b"item" ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ - "data", - b"data", - "item", - b"item", - "offset", - b"offset", - "superseded", - b"superseded", + "close", b"close", "data", b"data", "item", b"item", "offset", b"offset" ], ) -> None: ... def WhichOneof( self, oneof_group: typing_extensions.Literal["item", b"item"] - ) -> typing_extensions.Literal["data", "superseded"] | None: ... + ) -> typing_extensions.Literal["data", "close"] | None: ... global___StreamRecord = StreamRecord diff --git a/temporalio/api/streamservice/v1/request_response_pb2.py b/temporalio/api/streamservice/v1/request_response_pb2.py index 177dfb34e3..d4f1509ee8 100644 --- a/temporalio/api/streamservice/v1/request_response_pb2.py +++ b/temporalio/api/streamservice/v1/request_response_pb2.py @@ -24,7 +24,7 @@ ) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n4temporal/api/streamservice/v1/request_response.proto\x12\x1dtemporal.api.streamservice.v1\x1a\x1egoogle/protobuf/duration.proto\x1a"temporal/api/enums/v1/stream.proto\x1a$temporal/api/stream/v1/message.proto"\xdf\x02\n\x13\x43reateStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x35\n\nstream_ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12,\n\tretention\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x14\n\x0cwithdraw_run\x18\x04 \x01(\t\x12/\n\x05owner\x18\x05 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12\x16\n\x0e\x63lose_with_run\x18\x06 \x01(\x08\x12\x12\n\ntask_token\x18\x07 \x01(\x0c\x12I\n\x12id_conflict_policy\x18\x08 \x01(\x0e\x32-.temporal.api.enums.v1.StreamIdConflictPolicy\x12\x12\n\nrequest_id\x18\t \x01(\t"W\n\x14\x43reateStreamResponse\x12.\n\x03ref\x18\x01 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x0f\n\x07\x63reated\x18\x02 \x01(\x08"\x86\x02\n\x14\x41ppendRecordsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12.\n\x03ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x34\n\x06writer\x18\x03 \x01(\x0b\x32$.temporal.api.stream.v1.StreamWriter\x12\x10\n\x08sequence\x18\x04 \x01(\x03\x12\x35\n\x07records\x18\x05 \x03(\x0b\x32$.temporal.api.stream.v1.AppendRecord\x12\x18\n\x10\x64\x65\x66\x65r_withdrawal\x18\x06 \x01(\x08\x12\x12\n\ntask_token\x18\x07 \x01(\x0c"B\n\x15\x41ppendRecordsResponse\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03"\xa1\x01\n\x12\x43loseStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12.\n\x03ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x34\n\x06writer\x18\x03 \x01(\x0b\x32$.temporal.api.stream.v1.StreamWriter\x12\x12\n\ntask_token\x18\x04 \x01(\x0c"\x15\n\x13\x43loseStreamResponse"\xec\x02\n\x12ReadRecordsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12.\n\x03ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x03\x12\x35\n\x05start\x18\x04 \x01(\x0e\x32&.temporal.api.enums.v1.StreamReadStart\x12\x13\n\x0bmax_records\x18\x05 \x01(\x05\x12\'\n\x04wait\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12.\n\x0b\x63reate_wait\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x11\n\tto_offset\x18\x08 \x01(\x03\x12\x11\n\twake_only\x18\t \x01(\x08\x12\x0c\n\x04last\x18\n \x01(\x05\x12\x11\n\tmax_bytes\x18\x0b \x01(\x03\x12\x12\n\nfloor_from\x18\x0c \x01(\x03"\x92\x01\n\x13ReadRecordsResponse\x12\x35\n\x07records\x18\x01 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12!\n\x19\x63ontinue_as_new_suggested\x18\x04 \x01(\x08"Z\n\x15\x44\x65scribeStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12.\n\x03ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef"\x90\x01\n\x16\x44\x65scribeStreamResponse\x12\x33\n\x06status\x18\x01 \x01(\x0e\x32#.temporal.api.enums.v1.StreamStatus\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03\x12,\n\tretention\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationB\xb4\x01\n io.temporal.api.streamservice.v1B\x14RequestResponseProtoP\x01Z1go.temporal.io/api/streamservice/v1;streamservice\xaa\x02\x1fTemporalio.Api.StreamService.V1\xea\x02"Temporalio::Api::StreamService::V1b\x06proto3' + b'\n4temporal/api/streamservice/v1/request_response.proto\x12\x1dtemporal.api.streamservice.v1\x1a\x1egoogle/protobuf/duration.proto\x1a"temporal/api/enums/v1/stream.proto\x1a$temporal/api/stream/v1/message.proto"\x80\x02\n\x13\x43reateStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12\x35\n\nstream_ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12,\n\tretention\x18\x03 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x12\n\ntask_token\x18\x07 \x01(\x0c\x12I\n\x12id_conflict_policy\x18\x08 \x01(\x0e\x32-.temporal.api.enums.v1.StreamIdConflictPolicy\x12\x12\n\nrequest_id\x18\t \x01(\t"W\n\x14\x43reateStreamResponse\x12.\n\x03ref\x18\x01 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x0f\n\x07\x63reated\x18\x02 \x01(\x08"\xec\x01\n\x14\x41ppendRecordsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12.\n\x03ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x34\n\x06writer\x18\x03 \x01(\x0b\x32$.temporal.api.stream.v1.StreamWriter\x12\x10\n\x08sequence\x18\x04 \x01(\x03\x12\x35\n\x07records\x18\x05 \x03(\x0b\x32$.temporal.api.stream.v1.AppendRecord\x12\x12\n\ntask_token\x18\x07 \x01(\x0c"B\n\x15\x41ppendRecordsResponse\x12\x14\n\x0c\x66irst_offset\x18\x01 \x01(\x03\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03"\xa1\x01\n\x12\x43loseStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12.\n\x03ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x34\n\x06writer\x18\x03 \x01(\x0b\x32$.temporal.api.stream.v1.StreamWriter\x12\x12\n\ntask_token\x18\x04 \x01(\x0c"\x15\n\x13\x43loseStreamResponse"\xec\x02\n\x12ReadRecordsRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12.\n\x03ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef\x12\x13\n\x0b\x66rom_offset\x18\x03 \x01(\x03\x12\x35\n\x05start\x18\x04 \x01(\x0e\x32&.temporal.api.enums.v1.StreamReadStart\x12\x13\n\x0bmax_records\x18\x05 \x01(\x05\x12\'\n\x04wait\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12.\n\x0b\x63reate_wait\x18\x07 \x01(\x0b\x32\x19.google.protobuf.Duration\x12\x11\n\tto_offset\x18\x08 \x01(\x03\x12\x11\n\twake_only\x18\t \x01(\x08\x12\x0c\n\x04last\x18\n \x01(\x05\x12\x11\n\tmax_bytes\x18\x0b \x01(\x03\x12\x12\n\nfloor_from\x18\x0c \x01(\x03"\x92\x01\n\x13ReadRecordsResponse\x12\x35\n\x07records\x18\x01 \x03(\x0b\x32$.temporal.api.stream.v1.StreamRecord\x12\x13\n\x0bnext_offset\x18\x02 \x01(\x03\x12\x0c\n\x04\x64one\x18\x03 \x01(\x08\x12!\n\x19\x63ontinue_as_new_suggested\x18\x04 \x01(\x08"Z\n\x15\x44\x65scribeStreamRequest\x12\x11\n\tnamespace\x18\x01 \x01(\t\x12.\n\x03ref\x18\x02 \x01(\x0b\x32!.temporal.api.stream.v1.StreamRef"\x90\x01\n\x16\x44\x65scribeStreamResponse\x12\x33\n\x06status\x18\x01 \x01(\x0e\x32#.temporal.api.enums.v1.StreamStatus\x12\x13\n\x0blast_offset\x18\x02 \x01(\x03\x12,\n\tretention\x18\x03 \x01(\x0b\x32\x19.google.protobuf.DurationB\xb4\x01\n io.temporal.api.streamservice.v1B\x14RequestResponseProtoP\x01Z1go.temporal.io/api/streamservice/v1;streamservice\xaa\x02\x1fTemporalio.Api.StreamService.V1\xea\x02"Temporalio::Api::StreamService::V1b\x06proto3' ) @@ -152,23 +152,23 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n io.temporal.api.streamservice.v1B\024RequestResponseProtoP\001Z1go.temporal.io/api/streamservice/v1;streamservice\252\002\037Temporalio.Api.StreamService.V1\352\002"Temporalio::Api::StreamService::V1' _CREATESTREAMREQUEST._serialized_start = 194 - _CREATESTREAMREQUEST._serialized_end = 545 - _CREATESTREAMRESPONSE._serialized_start = 547 - _CREATESTREAMRESPONSE._serialized_end = 634 - _APPENDRECORDSREQUEST._serialized_start = 637 - _APPENDRECORDSREQUEST._serialized_end = 899 - _APPENDRECORDSRESPONSE._serialized_start = 901 - _APPENDRECORDSRESPONSE._serialized_end = 967 - _CLOSESTREAMREQUEST._serialized_start = 970 - _CLOSESTREAMREQUEST._serialized_end = 1131 - _CLOSESTREAMRESPONSE._serialized_start = 1133 - _CLOSESTREAMRESPONSE._serialized_end = 1154 - _READRECORDSREQUEST._serialized_start = 1157 - _READRECORDSREQUEST._serialized_end = 1521 - _READRECORDSRESPONSE._serialized_start = 1524 - _READRECORDSRESPONSE._serialized_end = 1670 - _DESCRIBESTREAMREQUEST._serialized_start = 1672 - _DESCRIBESTREAMREQUEST._serialized_end = 1762 - _DESCRIBESTREAMRESPONSE._serialized_start = 1765 - _DESCRIBESTREAMRESPONSE._serialized_end = 1909 + _CREATESTREAMREQUEST._serialized_end = 450 + _CREATESTREAMRESPONSE._serialized_start = 452 + _CREATESTREAMRESPONSE._serialized_end = 539 + _APPENDRECORDSREQUEST._serialized_start = 542 + _APPENDRECORDSREQUEST._serialized_end = 778 + _APPENDRECORDSRESPONSE._serialized_start = 780 + _APPENDRECORDSRESPONSE._serialized_end = 846 + _CLOSESTREAMREQUEST._serialized_start = 849 + _CLOSESTREAMREQUEST._serialized_end = 1010 + _CLOSESTREAMRESPONSE._serialized_start = 1012 + _CLOSESTREAMRESPONSE._serialized_end = 1033 + _READRECORDSREQUEST._serialized_start = 1036 + _READRECORDSREQUEST._serialized_end = 1400 + _READRECORDSRESPONSE._serialized_start = 1403 + _READRECORDSRESPONSE._serialized_end = 1549 + _DESCRIBESTREAMREQUEST._serialized_start = 1551 + _DESCRIBESTREAMREQUEST._serialized_end = 1641 + _DESCRIBESTREAMRESPONSE._serialized_start = 1644 + _DESCRIBESTREAMRESPONSE._serialized_end = 1788 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/streamservice/v1/request_response_pb2.pyi b/temporalio/api/streamservice/v1/request_response_pb2.pyi index f5becf5a53..3bb48818c9 100644 --- a/temporalio/api/streamservice/v1/request_response_pb2.pyi +++ b/temporalio/api/streamservice/v1/request_response_pb2.pyi @@ -26,8 +26,8 @@ class CreateStreamRequest(google.protobuf.message.Message): """Experimental. Might change in the future. Creates the Stream. When the Stream exists, FAIL fails with ALREADY_EXISTS, and USE_EXISTING gets it with created false and its own Retention. A retry of the create that made the Stream, - with the same request_id, gets created true. Each policy fails with CLOSED on a closed Stream, - EXPIRED on an expired one, and NOT_OWNER on one with another owner. + with the same request_id, gets created true. Each policy fails with CLOSED on a closed Stream + and EXPIRED on an expired one. (-- api-linter: core::0133=disabled aip.dev/not-precedent: The request identifies a Stream by namespace and ref. --) """ @@ -37,39 +37,20 @@ class CreateStreamRequest(google.protobuf.message.Message): NAMESPACE_FIELD_NUMBER: builtins.int STREAM_REF_FIELD_NUMBER: builtins.int RETENTION_FIELD_NUMBER: builtins.int - WITHDRAW_RUN_FIELD_NUMBER: builtins.int - OWNER_FIELD_NUMBER: builtins.int - CLOSE_WITH_RUN_FIELD_NUMBER: builtins.int TASK_TOKEN_FIELD_NUMBER: builtins.int ID_CONFLICT_POLICY_FIELD_NUMBER: builtins.int REQUEST_ID_FIELD_NUMBER: builtins.int namespace: builtins.str @property def stream_ref(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: - """The ref of the Stream to create. Its id is the Stream id, and is never empty. A bound - Stream's ref has the owner's run chain as its owner. - """ + """The ref of the Stream to create. Its id is the Stream id, and is never empty.""" @property def retention(self) -> google.protobuf.duration_pb2.Duration: """The age after which a Record is dropped. Empty is the Store default.""" - withdraw_run: builtins.str - """Workflow only. The previous run on a retry run's first create, to withdraw its Records.""" - @property - def owner(self) -> temporalio.api.stream.v1.message_pb2.Producer: - """The Workflow run that owns a bound Stream, with its run_id and a Workflow epoch. It needs a - ref whose owner is the run's chain. Unset for a standalone Stream. - """ - close_with_run: builtins.bool - """Only read by the server. When set, the server closes the Stream when the owner's run chain - ends, also on a terminate or a timeout. It needs owner, and a ref with a Workflow owner. - (-- api-linter: core::0140::prepositions=disabled - aip.dev/not-precedent: "with" ties the close to the owner's run chain. --) - """ task_token: builtins.bytes """Only read by the server, for a Stream bound to a Standalone Activity. The server checks that the token is the Activity's current attempt, so a caller can't bind a Stream to a Standalone - Activity that it isn't. The server knows the run of a Workflow's call, and a standalone - Stream has no owner, so neither needs a token. + Activity that it isn't. """ id_conflict_policy: ( temporalio.api.enums.v1.stream_pb2.StreamIdConflictPolicy.ValueType @@ -85,9 +66,6 @@ class CreateStreamRequest(google.protobuf.message.Message): namespace: builtins.str = ..., stream_ref: temporalio.api.stream.v1.message_pb2.StreamRef | None = ..., retention: google.protobuf.duration_pb2.Duration | None = ..., - withdraw_run: builtins.str = ..., - owner: temporalio.api.stream.v1.message_pb2.Producer | None = ..., - close_with_run: builtins.bool = ..., task_token: builtins.bytes = ..., id_conflict_policy: temporalio.api.enums.v1.stream_pb2.StreamIdConflictPolicy.ValueType = ..., request_id: builtins.str = ..., @@ -95,20 +73,16 @@ class CreateStreamRequest(google.protobuf.message.Message): def HasField( self, field_name: typing_extensions.Literal[ - "owner", b"owner", "retention", b"retention", "stream_ref", b"stream_ref" + "retention", b"retention", "stream_ref", b"stream_ref" ], ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ - "close_with_run", - b"close_with_run", "id_conflict_policy", b"id_conflict_policy", "namespace", b"namespace", - "owner", - b"owner", "request_id", b"request_id", "retention", @@ -117,8 +91,6 @@ class CreateStreamRequest(google.protobuf.message.Message): b"stream_ref", "task_token", b"task_token", - "withdraw_run", - b"withdraw_run", ], ) -> None: ... @@ -135,7 +107,7 @@ class CreateStreamResponse(google.protobuf.message.Message): CREATED_FIELD_NUMBER: builtins.int @property def ref(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: - """The ref, with the owner of a bound Stream.""" + """The ref of the Stream.""" created: builtins.bool """False when USE_EXISTING got a Stream that existed.""" def __init__( @@ -167,7 +139,6 @@ class AppendRecordsRequest(google.protobuf.message.Message): WRITER_FIELD_NUMBER: builtins.int SEQUENCE_FIELD_NUMBER: builtins.int RECORDS_FIELD_NUMBER: builtins.int - DEFER_WITHDRAWAL_FIELD_NUMBER: builtins.int TASK_TOKEN_FIELD_NUMBER: builtins.int namespace: builtins.str @property @@ -185,10 +156,6 @@ class AppendRecordsRequest(google.protobuf.message.Message): ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ temporalio.api.stream.v1.message_pb2.AppendRecord ]: ... - defer_withdrawal: builtins.bool - """Only read by the server, on a Workflow Producer's batch. The batch withdraws no Workflow - Task and keeps no task facts, but the writer's withdraw_run still applies. - """ task_token: builtins.bytes """Only read by the server. The server checks the Producer against the Activity's current attempt, so nobody else writes as that Activity. @@ -204,7 +171,6 @@ class AppendRecordsRequest(google.protobuf.message.Message): temporalio.api.stream.v1.message_pb2.AppendRecord ] | None = ..., - defer_withdrawal: builtins.bool = ..., task_token: builtins.bytes = ..., ) -> None: ... def HasField( @@ -213,8 +179,6 @@ class AppendRecordsRequest(google.protobuf.message.Message): def ClearField( self, field_name: typing_extensions.Literal[ - "defer_withdrawal", - b"defer_withdrawal", "namespace", b"namespace", "records", @@ -262,8 +226,6 @@ class CloseStreamRequest(google.protobuf.message.Message): """Experimental. Might change in the future. Closes a Stream. Readers drain what is left, then end, and later appends fail with CLOSED. A close writes no Record and carries no value, so code that has a last value publishes it first. - Only the owner's run chain closes a bound Stream. Only a Workflow or a client closes a - standalone Stream. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -277,9 +239,7 @@ class CloseStreamRequest(google.protobuf.message.Message): def ref(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... @property def writer(self) -> temporalio.api.stream.v1.message_pb2.StreamWriter: - """The writer that closes the Stream. An SDK Store reads only its Producer, and the server - also reads its withdrawal fields. - """ + """The writer that closes the Stream. A Store reads only its Producer.""" task_token: builtins.bytes """Only read by the server. It checks the token as on the append.""" def __init__( @@ -354,9 +314,7 @@ class ReadRecordsRequest(google.protobuf.message.Message): start: temporalio.api.enums.v1.stream_pb2.StreamReadStart.ValueType """Where to start when from_offset is zero. Unspecified is the beginning.""" max_records: builtins.int - """The most items in one page, Superseded items included. Zero is 1000. A Superseded item and - the Record after it stay in one page, so a page of one Record can hold two items. - """ + """The most Records in one page. Zero is 1000.""" @property def wait(self) -> google.protobuf.duration_pb2.Duration: """How long the read waits for a new Record before it answers with an empty page. Empty answers @@ -542,7 +500,7 @@ class DescribeStreamResponse(google.protobuf.message.Message): RETENTION_FIELD_NUMBER: builtins.int status: temporalio.api.enums.v1.stream_pb2.StreamStatus.ValueType last_offset: builtins.int - """The newest offset, of a Record or a Superseded item. Zero when the Stream has none.""" + """The newest offset, of a Record. Zero when the Stream has none.""" @property def retention(self) -> google.protobuf.duration_pb2.Duration: ... def __init__( diff --git a/temporalio/bridge/proto/streams/streams_pb2.pyi b/temporalio/bridge/proto/streams/streams_pb2.pyi index b9b9655523..25934fe205 100644 --- a/temporalio/bridge/proto/streams/streams_pb2.pyi +++ b/temporalio/bridge/proto/streams/streams_pb2.pyi @@ -95,9 +95,10 @@ class OpenWriterRequest(google.protobuf.message.Message): def ref(self) -> temporalio.api.stream.v1.message_pb2.StreamRef: ... @property def writer(self) -> temporalio.api.stream.v1.message_pb2.StreamWriter: - """Each batch sends it as AppendRecords does. For a Local Activity it has the run's task facts, - so the writes withdraw the run's orphaned Local Activity Records. Its withdraw_run must be - empty, since only a retry run's first append withdraws a run, and a writer sends many. + """Each batch sends it as AppendRecords does. For a Local Activity its withdrawal has the run's + task facts, so the writes withdraw the run's orphaned Local Activity Records. The withdrawal's + withdraw_run must be empty, since only a retry run's first append withdraws a run, and a + writer sends many. """ max_batch_size: builtins.int """A write sends at most this many Records in one batch. Zero, or a value above 1000, is 1000.""" diff --git a/temporalio/client/_client.py b/temporalio/client/_client.py index 25997af2df..e0d296e6fe 100644 --- a/temporalio/client/_client.py +++ b/temporalio/client/_client.py @@ -113,7 +113,6 @@ if TYPE_CHECKING: from temporalio.streams import ( - Execution, StreamBackend, StreamIDConflictPolicy, StreamRef, @@ -1016,7 +1015,6 @@ def get_stream( /, *, payload_type: type[_StreamPayloadT], - owner: Execution | None = None, backend: str | None = None, ) -> Stream[_StreamPayloadT]: ... @@ -1027,7 +1025,6 @@ def get_stream( /, *, payload_type: None = None, - owner: Execution | None = None, backend: str | None = None, ) -> Stream[Any]: ... @@ -1037,7 +1034,6 @@ def get_stream( /, *, payload_type: type[Any] | None = None, - owner: Execution | None = None, backend: str | None = None, ) -> Stream[Any]: """Get a Stream by its id, or by a ref that came as data. It makes no call. @@ -1050,7 +1046,6 @@ def get_stream( Args: id_or_ref: The Stream id, or a ref of the Stream. payload_type: The type that reads decode each Record with, for the id form. - owner: The owner of a bound Stream, for the id form. backend: The name of the backend, for the id form. Defaults to the registered backend with ``default=True``. @@ -1066,7 +1061,7 @@ def get_stream( if isinstance(id_or_ref, str): return _stream_client.get_stream( - self, payload_type, id=id_or_ref, owner=owner, backend=backend + self, payload_type, id=id_or_ref, backend=backend ) return _stream_client.from_ref(self, id_or_ref) diff --git a/temporalio/client/_streams.py b/temporalio/client/_streams.py index 62b5373ba1..60330c7889 100644 --- a/temporalio/client/_streams.py +++ b/temporalio/client/_streams.py @@ -19,12 +19,12 @@ if TYPE_CHECKING: import temporalio.api.stream.v1 from temporalio.streams import ( + Closed, End, Reader, Record, StreamOffset, StreamRef, - Superseded, ) from temporalio.streams._stream_client import StreamCalls @@ -231,7 +231,7 @@ def __init__(self, reader: Reader[T]) -> None: from temporalio.streams import _activity_stream self._reader = reader - self._steps: Generator[Record[T] | Superseded, None, None] = ( + self._steps: Generator[Record[T] | Closed, None, None] = ( _activity_stream.sync_items(reader) ) @@ -244,7 +244,7 @@ def __iter__(self) -> SyncReader[T]: """The read is its own iterator.""" return self - def __next__(self) -> Record[T] | Superseded: + def __next__(self) -> Record[T] | Closed: """The next item.""" return next(self._steps) diff --git a/temporalio/nexus/system/_support.py b/temporalio/nexus/system/_support.py index 7e6311ad6f..e1daa932ee 100644 --- a/temporalio/nexus/system/_support.py +++ b/temporalio/nexus/system/_support.py @@ -364,14 +364,8 @@ def offset(self) -> int | None: ... def resume_read_request(request: _ReadRequestT, item: _ReadItem) -> _ReadRequestT: """The request that resumes a Stream read after ``item``.""" - from temporalio.streams.system.models import StreamItemSuperseded - - offset = item.offset or 0 - if not isinstance(item.item, StreamItemSuperseded): - offset += 1 - # A Superseded item has the offset of the first Record of its batch, which comes next. So a - # read resumes at that offset, and gets the Superseded item again, which withdraws nothing - # new. The first page's start and last no longer apply. + # A read resumes past the last item. The first page's start and last no longer apply. + offset = (item.offset or 0) + 1 return dataclasses.replace(request, from_offset=offset, start=None, last=None) @@ -477,22 +471,7 @@ def workflow_writer() -> typing.Any: from temporalio.streams.system import models from temporalio.workflow import info - return models.StreamWriter( - producer=_ids.workflow_producer(info().run_id), - failed_tasks=[], - withdraw_run=withdraw_run(), - ) - - -def withdraw_run() -> str | None: - """The previous run whose Records a retry run's first write withdraws.""" - from temporalio.workflow import in_workflow, info - - if not in_workflow(): - # Code outside a Workflow isn't a retry run. - return None - run = info() - return run.continued_run_id if run.attempt > 1 and run.continued_run_id else None + return models.StreamWriter(producer=_ids.workflow_producer(info().run_id)) def client_writer() -> typing.Any: @@ -506,12 +485,7 @@ def client_writer() -> typing.Any: # Each client has its own Producer, and a generated call doesn't say which client makes # it. So the SDK's Streams send their writer, and a call without one fails. raise RuntimeError("A Stream write outside a Workflow needs its Producer") - writer = stream_v1.StreamWriter( - producer=streams.producer, - # Only a Local Activity has the run's task facts. - previous_completed_task=streams.previous_completed_task, - failed_tasks=streams.failed_tasks, - ) + writer = stream_v1.StreamWriter(producer=streams.producer) return models._StreamWriterTransferTypeConverter().from_transfer_type( writer, models.StreamWriter ) diff --git a/temporalio/streams/__init__.py b/temporalio/streams/__init__.py index 845f6ea451..25fad6e6ae 100644 --- a/temporalio/streams/__init__.py +++ b/temporalio/streams/__init__.py @@ -19,9 +19,11 @@ :py:func:`temporalio.workflow.get_stream` and :py:meth:`temporalio.client.Client.get_stream` give the Stream of a ref. In an Activity, a Stream from ``activity.client()`` writes as the attempt. -A read gives :py:class:`Record` and :py:class:`Superseded` items, and the ``kind`` of each item -narrows its type. A :py:class:`StreamOffset` is the opaque position of a Record. A close carries -no value. A read of a closed Stream ends once it gives the last item. +A read gives :py:class:`Record` items and a :py:class:`Closed` item last, and the ``kind`` of each +item narrows its type. A :py:class:`StreamOffset` is the opaque position of a Record. For +supersession, the opt-in :py:func:`temporalio.streams.superseded.with_superseded` wraps a reader +and adds ``Superseded`` items from the Producer epochs. A read of a closed Stream ends after its +``Closed`` item. A :py:class:`StreamBackend` makes the Stream calls and registers a backend name. Examples are :py:class:`temporalio.contrib.stream.RedisStreamBackend` (``redis``) and @@ -36,6 +38,7 @@ # The reason of a StreamError is the proto enum, as in the StreamErrorDetails of a failed call. from temporalio.api.enums.v1 import StreamErrorReason +from temporalio.streams._close import StreamCloseStatus from temporalio.streams._conflict import StreamIDConflictPolicy from temporalio.streams._errors import ( StreamAlreadyExistsError, @@ -45,55 +48,59 @@ StreamFencedError, StreamNotFoundError, ) -from temporalio.streams._items import END, End, ItemKind, Record, Superseded +from temporalio.streams._items import END, Closed, End, ItemKind, Record from temporalio.streams._offset import StreamOffset from temporalio.streams._plugin import StreamBackend from temporalio.streams._reader import Reader from temporalio.streams._ref import StreamRef from temporalio.streams._stream_client import producer -# nexgen generates these wire models. Refs, Records and Superseded items carry them. +# nexgen generates these wire models. Refs, Records and close items carry them. from temporalio.streams.system.models import ( ActivityEpoch, + ActivityProducer, ClientEpoch, - Execution, - ExecutionType, + ClientProducer, LocalActivityEpoch, + LocalActivityProducer, Producer, - ProducerEpoch, - ProducerEpochKind, - ProducerEpochKindActivity, - ProducerEpochKindClient, - ProducerEpochKindLocalActivity, - ProducerEpochKindStandaloneActivity, - ProducerEpochKindWorkflow, - StreamSuperseded, + ProducerKind, + ProducerKindActivity, + ProducerKindClient, + ProducerKindLocalActivity, + ProducerKindStandaloneActivity, + ProducerKindWorkflow, + StreamClose, StreamWriter, Timestamp, + WorkflowProducer, WorkflowTaskEpoch, ) __all__ = [ "END", "ActivityEpoch", + "ActivityProducer", "ClientEpoch", + "ClientProducer", + "Closed", "End", - "Execution", - "ExecutionType", "ItemKind", "LocalActivityEpoch", + "LocalActivityProducer", "Producer", - "ProducerEpoch", - "ProducerEpochKind", - "ProducerEpochKindActivity", - "ProducerEpochKindClient", - "ProducerEpochKindLocalActivity", - "ProducerEpochKindStandaloneActivity", - "ProducerEpochKindWorkflow", + "ProducerKind", + "ProducerKindActivity", + "ProducerKindClient", + "ProducerKindLocalActivity", + "ProducerKindStandaloneActivity", + "ProducerKindWorkflow", "Reader", "Record", "StreamAlreadyExistsError", "StreamBackend", + "StreamClose", + "StreamCloseStatus", "StreamClosedError", "StreamConfigError", "StreamError", @@ -103,10 +110,9 @@ "StreamNotFoundError", "StreamOffset", "StreamRef", - "StreamSuperseded", "StreamWriter", - "Superseded", "Timestamp", + "WorkflowProducer", "WorkflowTaskEpoch", "producer", ] diff --git a/temporalio/streams/_access.py b/temporalio/streams/_access.py index 2aec4baca2..1b455c4e56 100644 --- a/temporalio/streams/_access.py +++ b/temporalio/streams/_access.py @@ -82,7 +82,7 @@ async def close(self) -> None: await service.send_close_stream( models.CloseStreamRequest( ref=models.StreamRef(id=self.stream_id), - writer=models.StreamWriter(producer=producer, failed_tasks=[]), + writer=models.StreamWriter(producer=producer), ) ) diff --git a/temporalio/streams/_activity.py b/temporalio/streams/_activity.py index 1b900804be..f4b1e81611 100644 --- a/temporalio/streams/_activity.py +++ b/temporalio/streams/_activity.py @@ -26,11 +26,8 @@ class ActivityStreams: namespace: str producer: stream_v1.Producer loop: asyncio.AbstractEventLoop - # The attempt's Workflow ID, which the close Activity checks against a bound Stream's owner. + # The attempt's Workflow ID. workflow_id: str | None = None - # A Local Activity's writes use these to withdraw the run's orphaned Records. - previous_completed_task: stream_v1.WorkflowTaskEpoch | None = None - failed_tasks: tuple[int, ...] = () writers: dict[str, RecordWriter] = field(default_factory=dict) # The attempt's BufferedAppenders, which the end of the attempt flushes. appenders: list[BufferedAppender[Any]] = field(default_factory=list) @@ -47,8 +44,6 @@ def writer(self, stream_id: str) -> RecordWriter: stream_id=stream_id, producer=self.producer, free=self.backend.free_writer, - previous_completed_task=self.previous_completed_task, - failed_tasks=self.failed_tasks, ) self.writers[stream_id] = writer return writer diff --git a/temporalio/streams/_activity_stream.py b/temporalio/streams/_activity_stream.py index 863983a529..18fc9f51fd 100644 --- a/temporalio/streams/_activity_stream.py +++ b/temporalio/streams/_activity_stream.py @@ -11,7 +11,7 @@ from temporalio.client import Stream from temporalio.streams import _activity, _backend, _errors from temporalio.streams._access import Access, ReadSpec, reader -from temporalio.streams._items import Record, Superseded +from temporalio.streams._items import Closed, Record from temporalio.streams._offset import StreamOffset from temporalio.streams._reader import Reader from temporalio.streams._ref import StreamRef @@ -198,7 +198,7 @@ def run_blocking(call: Coroutine[Any, Any, _R]) -> _R: def sync_items( items: Reader[T], -) -> Generator[Record[T] | Superseded, None, None]: +) -> Generator[Record[T] | Closed, None, None]: """The items of a read, each of which blocks the Activity's thread.""" try: while True: @@ -211,5 +211,5 @@ def sync_items( run_blocking(items.aclose()) -async def _next(items: Reader[T]) -> Record[T] | Superseded: +async def _next(items: Reader[T]) -> Record[T] | Closed: return await items.__anext__() diff --git a/temporalio/streams/_close.py b/temporalio/streams/_close.py new file mode 100644 index 0000000000..2a9ecaecb9 --- /dev/null +++ b/temporalio/streams/_close.py @@ -0,0 +1,39 @@ +"""The close status of a Stream.""" + +from __future__ import annotations + +from enum import IntEnum + +import temporalio.api.enums.v1 + + +class StreamCloseStatus(IntEnum): + """Why a Stream closed, on a :py:class:`temporalio.streams.Closed` item. + + The memory and Redis backends give ``CLOSED``. The owner statuses and a close failure come + from the native runtime. See :py:class:`temporalio.api.enums.v1.StreamCloseStatus`. + + .. warning:: + This API is experimental. + """ + + UNSPECIFIED = int( + temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_UNSPECIFIED + ) + CLOSED = int(temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_CLOSED) + OWNER_COMPLETED = int( + temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_COMPLETED + ) + OWNER_FAILED = int( + temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_FAILED + ) + OWNER_CANCELED = int( + temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_CANCELED + ) + OWNER_TERMINATED = int( + temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_TERMINATED + ) + OWNER_TIMED_OUT = int( + temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_TIMED_OUT + ) + IDLE = int(temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_IDLE) diff --git a/temporalio/streams/_extension.py b/temporalio/streams/_extension.py index 9299bcce37..806fc6a571 100644 --- a/temporalio/streams/_extension.py +++ b/temporalio/streams/_extension.py @@ -445,12 +445,6 @@ async def execute_activity( streams = _activity_streams(input, self._interceptor) token = _activity.current.set(streams) try: - if temporalio.activity.info().attempt > 1: - # Withdraw the Records of earlier attempts, even if this attempt publishes - # nothing. This creates no Stream. - stream_ids = [ref.id for ref in _stream_arguments(input.args)] - for stream_id in dict.fromkeys(stream_ids): - await streams.writer(stream_id).withdraw_earlier_attempts() result = await super().execute_activity(input) except BaseException: await _close_quietly(streams) @@ -518,20 +512,12 @@ def _activity_streams( converter = temporalio.converter.PayloadConverter.default key = input.headers.get(_KEY_HEADER) key_text = converter.from_payload(key, str) if key is not None else "" - previous: stream_v1.WorkflowTaskEpoch | None = None - failed: tuple[int, ...] = () if info.is_local: local = stream_v1.LocalActivityEpoch(attempt=info.attempt) task = info.workflow_task if task is not None: - # Always set it, with 0 if no task completed. If it's unset, the Store doesn't - # withdraw the run's orphaned Local Activity Records. - previous = stream_v1.WorkflowTaskEpoch( - started_event_id=task.previous_started_event_id - ) - failed = tuple(task.failed_tasks) - # The dispatching task orders the epoch first, so a later task's write withdraws a - # failed task's Records. + # The dispatching task orders the epoch first, so a later task's write has a higher + # epoch than a failed task's. local.task.started_event_id = task.started_event_id local.task.attempt = task.attempt if task.started_time is not None: @@ -539,8 +525,7 @@ def _activity_streams( run_id = info.workflow_run_id or "" producer = stream_v1.Producer( id=_ids.local_activity_producer_id(run_id, key_text), - epoch=stream_v1.ProducerEpoch(local_activity=local), - run_id=run_id, + local_activity=stream_v1.LocalActivityProducer(run_id=run_id, epoch=local), ) else: attempt = stream_v1.ActivityEpoch(attempt=info.attempt) @@ -549,15 +534,14 @@ def _activity_streams( run_id = info.workflow_run_id or "" producer = stream_v1.Producer( id=_ids.activity_producer_id(run_id, info.activity_id, key_text), - epoch=stream_v1.ProducerEpoch(activity=attempt), - run_id=run_id, + activity=stream_v1.ActivityProducer(run_id=run_id, epoch=attempt), ) else: producer = stream_v1.Producer( id=_ids.standalone_activity_producer_id( info.activity_id, info.activity_run_id or "" ), - epoch=stream_v1.ProducerEpoch(standalone_activity=attempt), + standalone_activity=stream_v1.ActivityProducer(epoch=attempt), ) return _activity.ActivityStreams( backend=interceptor.backend, @@ -566,7 +550,5 @@ def _activity_streams( producer=producer, loop=asyncio.get_running_loop(), workflow_id=info.workflow_id, - previous_completed_task=previous, - failed_tasks=failed, client=interceptor.client, ) diff --git a/temporalio/streams/_ids.py b/temporalio/streams/_ids.py index bf200c2a97..5c16837905 100644 --- a/temporalio/streams/_ids.py +++ b/temporalio/streams/_ids.py @@ -30,9 +30,8 @@ def workflow_producer(run_id: str) -> models.Producer: """ return models.Producer( id=workflow_producer_id(run_id), - run_id=run_id, - epoch=models.ProducerEpoch( - kind=models.ProducerEpochKindWorkflow(models.WorkflowTaskEpoch()) + kind=models.ProducerKindWorkflow( + models.WorkflowProducer(run_id=run_id, epoch=models.WorkflowTaskEpoch()) ), ) diff --git a/temporalio/streams/_items.py b/temporalio/streams/_items.py index 881aaaa052..7ef1b86a6c 100644 --- a/temporalio/streams/_items.py +++ b/temporalio/streams/_items.py @@ -4,8 +4,9 @@ import enum from dataclasses import dataclass, field -from typing import Any, Generic, Literal, TypeVar +from typing import Generic, Literal, TypeVar +from temporalio.streams._close import StreamCloseStatus from temporalio.streams._offset import StreamOffset from temporalio.streams.system import models @@ -31,8 +32,8 @@ class ItemKind(enum.Enum): RECORD = "record" """A :py:class:`Record`.""" - SUPERSEDED = "superseded" - """A :py:class:`Superseded` item.""" + CLOSED = "closed" + """A :py:class:`Closed` item, the last item of a read.""" @dataclass(frozen=True) @@ -57,162 +58,25 @@ class Record(Generic[T]): @dataclass(frozen=True) -class Superseded: - """An item that withdraws the Records of a failed attempt. +class Closed: + """The close of a Stream, the last item of a read. - Every reader gets it at the same point in the Stream. A live view removes the Records in - :py:attr:`withdrawn_offsets`. A view that keeps Records from an earlier read, such as from - before a reconnect, checks them with :py:meth:`withdraws`. - - The next write of a run to a Stream withdraws the Records that the run's Local Activities - wrote there in Workflow Tasks that never completed. A Stream that the run never writes - again keeps them. + A read of a closed Stream ends after this item. On the memory and Redis backends the status is + :py:attr:`StreamCloseStatus.StreamCloseStatusClosed` and ``failure`` is ``None``. The owner + statuses and a close failure come from the native runtime. .. warning:: This API is experimental. """ - kind: Literal[ItemKind.SUPERSEDED] = field(default=ItemKind.SUPERSEDED, init=False) - """Always :py:attr:`ItemKind.SUPERSEDED`.""" + kind: Literal[ItemKind.CLOSED] = field(default=ItemKind.CLOSED, init=False) + """Always :py:attr:`ItemKind.CLOSED`.""" offset: StreamOffset - """Position of the item in the Stream.""" - - facts: models.StreamSuperseded - """The withdrawal facts from the Store. Its ``writer`` gives the Records that it withdraws.""" - - withdrawn_offsets: tuple[StreamOffset, ...] = () - """The offsets of the Records that this read gave and this item withdraws, in order. - - A read gives each offset once. It doesn't include Records from before the read's start. - """ - - def withdraws(self, record: Record[Any] | models.Producer) -> bool: - """Whether this item withdraws an earlier Record, or the Records of a Producer. - - It answers for any earlier Record, such as one that an app showed before a resume, - from the Producer and epoch that each Record carries. So an app can keep only the - Producer of each Record that it shows. - """ - producer = record.producer if isinstance(record, Record) else record - return withdraws(self.facts, producer) - - -# The epoch rules match the crate's epoch keys. A field compares in the order of the key, and an -# empty or negative field compares as zero. -def _n(value: int | None) -> int: - return max(value or 0, 0) - - -def _time(value: models.Timestamp | None) -> tuple[int, int]: - if value is None: - return (0, 0) - return (_n(value.seconds), _n(value.nanos)) - - -_TaskKey = tuple[int, int, tuple[int, int]] - - -def _task_key(task: models.WorkflowTaskEpoch | None) -> _TaskKey: - if task is None: - return (0, 0, (0, 0)) - return (_n(task.started_event_id), _n(task.attempt), _time(task.started_time)) + """The offset at which the Stream ended.""" + status: StreamCloseStatus + """Why the Stream closed.""" -def _task_of(epoch: models.ProducerEpoch) -> models.WorkflowTaskEpoch | None: - # A Workflow's own task, or the task that dispatched a Local Activity. The other kinds have - # none. - match epoch.kind: - case models.ProducerEpochKindWorkflow(value=task): - return task - case models.ProducerEpochKindLocalActivity(value=local): - return local.task - case _: - return None - - -def _epoch_key(epoch: models.ProducerEpoch) -> tuple[Any, ...]: - scheduled: models.Timestamp | None = None - attempt: int | None = None - match epoch.kind: - case models.ProducerEpochKindLocalActivity(value=local): - attempt = local.attempt - case ( - models.ProducerEpochKindActivity(value=activity) - | models.ProducerEpochKindStandaloneActivity(value=activity) - ): - scheduled, attempt = activity.attempt_scheduled_time, activity.attempt - case models.ProducerEpochKindClient(value=client): - attempt = client.attempt - case _: - pass - return (*_task_key(_task_of(epoch)), _time(scheduled), _n(attempt)) - - -def _kind(epoch: models.ProducerEpoch | None) -> type[Any] | None: - return type(epoch.kind) if epoch is not None and epoch.kind is not None else None - - -def is_client(producer: models.Producer) -> bool: - """Whether the Producer is a client, whose newer epoch withdraws nothing.""" - return _kind(producer.epoch) is models.ProducerEpochKindClient - - -def producer_key(producer: models.Producer) -> tuple[Any, ...]: - """The Producer fields that :py:func:`withdraws` reads, so equal keys are withdrawn together.""" - epoch = _epoch_key(producer.epoch) if producer.epoch is not None else None - return (_kind(producer.epoch), producer.id, producer.run_id or "", epoch) - - -def _of_run(producer: models.Producer, by: models.Producer) -> bool: - # The run's Workflow Producer, or one of its Local Activities. - kind = _kind(producer.epoch) - if kind is models.ProducerEpochKindWorkflow: - return producer.id == by.id - if kind is models.ProducerEpochKindLocalActivity: - return (producer.run_id or "") == (by.run_id or "") - return False - - -def withdraws(facts: models.StreamSuperseded, producer: models.Producer | None) -> bool: - """Whether a Superseded item withdraws a Record of ``producer``. - - The rule matches the crate's, so every SDK withdraws the same Records. - """ - writer = facts.writer - if producer is None or writer is None: - return False - if writer.withdraw_run and producer.run_id == writer.withdraw_run: - return True - by, old = writer.producer, producer.epoch - epoch = by.epoch if by is not None else None - if by is None or epoch is None or old is None: - return False - kind = _kind(epoch) - if kind is None or kind is models.ProducerEpochKindClient: - # A client's newer epoch only fences. Its earlier Records are real messages, not a - # failed attempt's. - return False - if kind is not models.ProducerEpochKindWorkflow: - return ( - _kind(old) is kind - and producer.id == by.id - and _epoch_key(old) < _epoch_key(epoch) - ) - if not _of_run(producer, by): - return False - task, old_task = _task_of(epoch), _task_of(old) - started = (old_task.started_event_id if old_task else None) or 0 - if started in (writer.failed_tasks or []): - return True - completed = writer.previous_completed_task - after = (completed.started_event_id if completed else None) or 0 - if after < started < ((task.started_event_id if task else None) or 0): - return True - # No task in the range can have completed, so a lower attempt at either end goes too. - ends = [task] if completed is None else [task, completed] - return any( - started == ((end.started_event_id if end else None) or 0) - and _task_key(old_task) < _task_key(end) - for end in ends - ) + failure: BaseException | None = None + """The owner's failure for ``OWNER_FAILED``, or ``None``.""" diff --git a/temporalio/streams/_reader.py b/temporalio/streams/_reader.py index 9c4c3a2606..213fd41873 100644 --- a/temporalio/streams/_reader.py +++ b/temporalio/streams/_reader.py @@ -2,7 +2,6 @@ from __future__ import annotations -import array import asyncio import dataclasses from collections.abc import AsyncGenerator @@ -12,20 +11,14 @@ import temporalio.common import temporalio.converter from temporalio.streams import _backend, _convert, _errors -from temporalio.streams._items import ( - End, - Record, - Superseded, - is_client, - producer_key, - withdraws, -) +from temporalio.streams._close import StreamCloseStatus +from temporalio.streams._items import Closed, End, Record from temporalio.streams._offset import StreamOffset from temporalio.streams.system import models T = TypeVar("T") -Item = Record[Any] | Superseded +Item = Record[Any] | Closed # The service waits at most 60 s per read, so a longer wait is a loop of reads. _MAX_WAIT = timedelta(seconds=60) @@ -46,9 +39,9 @@ class Position: class Reader(Generic[T]): """The items of one read of a Stream, in order. Iterate it with ``async for``. - It yields :py:class:`Record` items, and :py:class:`Superseded` items that withdraw earlier - Records. Each item's ``kind`` gives its type. A read of a closed Stream ends after the last - item. + It yields :py:class:`Record` items, and a :py:class:`Closed` item last when the Stream is + closed. Each item's ``kind`` gives its type. A read of a closed Stream ends after the + ``Closed`` item. .. warning:: This API is experimental. @@ -65,23 +58,14 @@ def __init__( self._position = position # A client read's rpc_metadata and rpc_timeout, for each call of each step. self._options = options - # The Producer and the offsets of each Record that this read gave, without values, so a - # Superseded item can list the offsets that it withdraws. Each write Local Activity of a - # run is its own Producer, so a read of a Workflow's writes keeps one entry for each - # write, not for each Record, and a Superseded item scans those entries. - self._shown: dict[ - tuple[Any, ...], tuple[models.Producer, array.array[int]] - ] = {} @property def next_offset(self) -> StreamOffset | None: """The offset to resume from, after the last item or the last page that the read got. - Pass it as ``offset`` to a later read. After a ``Superseded`` item, it's that item's own - offset, since the first Record of its batch has the same offset and comes next. A read - from there gives the ``Superseded`` item again, which withdraws nothing new. After a - Record that failed to decode, it's past that Record. It's None before the first page of - a read from ``END`` or of a read with ``last``. + Pass it as ``offset`` to a later read. After a Record that failed to decode, it's past + that Record. It's None before the first page of a read from ``END`` or of a read with + ``last``. """ wire = self._position.next_offset return StreamOffset._from_wire(wire) if wire is not None else None @@ -90,7 +74,7 @@ def __aiter__(self) -> Reader[T]: """The read is its own iterator.""" return self - async def __anext__(self) -> Record[T] | Superseded: + async def __anext__(self) -> Record[T] | Closed: """The next item. Raises: @@ -100,42 +84,9 @@ async def __anext__(self) -> Record[T] | Superseded: with _backend.using(self._options): item = await self._items.__anext__() offset = item.offset._wire - if isinstance(item, Superseded): - # The Record of the same entry has the same offset and comes next. - self._position.next_offset = offset - else: - self._position.next_offset = max( - self._position.next_offset or 0, offset + 1 - ) - if ( - isinstance(item, Record) - and item.producer is not None - # Nothing withdraws a client's Records, so a long read of them keeps no offsets. - and not is_client(item.producer) - ): - key = producer_key(item.producer) - _, runs = self._shown.setdefault(key, (item.producer, array.array("Q"))) - # A batch has consecutive offsets, so a run of them costs one first and last pair. - if runs and runs[-1] + 1 == offset: - runs[-1] = offset - else: - runs.extend((offset, offset)) - elif isinstance(item, Superseded): - item = dataclasses.replace( - item, withdrawn_offsets=self._withdraw(item.facts) - ) + self._position.next_offset = max(self._position.next_offset or 0, offset + 1) return item - def _withdraw(self, facts: models.StreamSuperseded) -> tuple[StreamOffset, ...]: - offsets: list[int] = [] - for key, (producer, runs) in list(self._shown.items()): - if withdraws(facts, producer): - for first, last in zip(runs[::2], runs[1::2]): - offsets.extend(range(first, last + 1)) - # A withdrawn Record stays withdrawn, so the read forgets it. - del self._shown[key] - return tuple(StreamOffset._from_wire(offset) for offset in sorted(offsets)) - async def aclose(self) -> None: """End the read early, from the task that reads it. It makes no more calls. @@ -316,5 +267,11 @@ async def _item( offset=StreamOffset._from_wire(offset), producer=data.producer, ) - case models.StreamItemSuperseded(value=facts): - return Superseded(offset=StreamOffset._from_wire(offset), facts=facts) + case models.StreamItemClose(value=close): + return Closed( + offset=StreamOffset._from_wire(offset), + status=StreamCloseStatus(int(close.status)) + if close.status is not None + else StreamCloseStatus.UNSPECIFIED, + failure=close.failure, + ) diff --git a/temporalio/streams/_ref.py b/temporalio/streams/_ref.py index d87414bcec..9efe05a620 100644 --- a/temporalio/streams/_ref.py +++ b/temporalio/streams/_ref.py @@ -16,7 +16,7 @@ def _model(ref: StreamRef[Any]) -> models.StreamRef: - return models.StreamRef(id=ref.id, owner=ref.owner, backend=ref.backend) + return models.StreamRef(id=ref.id, backend=ref.backend) def to_proto(ref: StreamRef[Any]) -> stream_v1.StreamRef: @@ -44,9 +44,7 @@ def _from_json(value: Any, type_hint: Any) -> StreamRef[Any]: model = models._StreamRefTransferTypeConverter().from_transfer_type( message, models.StreamRef ) - # A ref is data, so it keeps any owner, such as a Standalone Activity's. A close checks it. - owner = model.owner if model.owner != models.Execution() else None - return origin(id=model.id, owner=owner, backend=model.backend) + return origin(id=model.id, backend=model.backend) class _StreamRefJson( @@ -82,12 +80,10 @@ class StreamRef(Generic[T]): def __init__( self, id: str, - owner: models.Execution | None = None, backend: str | None = None, ) -> None: """Create a ref. Code gets one from a Stream's ``ref`` instead.""" self._id = id - self._owner = owner self._backend = backend @property @@ -95,11 +91,6 @@ def id(self) -> str: """The Stream id. It is opaque to callers.""" return self._id - @property - def owner(self) -> models.Execution | None: - """The owner of a bound Stream, such as a Workflow run chain. None for a standalone one.""" - return self._owner - @property def backend(self) -> str | None: """Name of the backend that stores the Stream. None means the default one.""" @@ -110,11 +101,7 @@ def __eq__(self, other: object) -> bool: if type(other) is not type(self): return NotImplemented assert isinstance(other, StreamRef) - return (self._id, self._owner, self._backend) == ( - other._id, - other._owner, - other._backend, - ) + return (self._id, self._backend) == (other._id, other._backend) def __hash__(self) -> int: """Hash of the id and the backend, which identify the Stream.""" @@ -122,10 +109,7 @@ def __hash__(self) -> int: def __repr__(self) -> str: """Show the fields.""" - return ( - f"{type(self).__name__}(id={self._id!r}, owner={self._owner!r}, " - f"backend={self._backend!r})" - ) + return f"{type(self).__name__}(id={self._id!r}, backend={self._backend!r})" def __copy__(self) -> Self: """A ref doesn't change, so a copy is the same ref.""" @@ -148,17 +132,6 @@ def __get_pydantic_core_schema__(cls, source: Any, handler: Any) -> Any: json_schema_input_schema=core_schema.typed_dict_schema( { "id": core_schema.typed_dict_field(core_schema.str_schema()), - "owner": core_schema.typed_dict_field( - core_schema.typed_dict_schema( - { - key: core_schema.typed_dict_field( - core_schema.str_schema(), required=False - ) - for key in ("type", "businessId", "runId") - } - ), - required=False, - ), "backend": core_schema.typed_dict_field( core_schema.str_schema(), required=False ), diff --git a/temporalio/streams/_stream_client.py b/temporalio/streams/_stream_client.py index 4a3d920bf8..9d288528f9 100644 --- a/temporalio/streams/_stream_client.py +++ b/temporalio/streams/_stream_client.py @@ -18,7 +18,7 @@ import temporalio.api.stream.v1 as stream_v1 import temporalio.client from temporalio.client._streams import Stream -from temporalio.streams import _activity, _backend, _errors, _ids +from temporalio.streams import _activity, _backend, _ids from temporalio.streams._access import Access, ReadSpec, reader from temporalio.streams._backend import BackendCalls from temporalio.streams._conflict import StreamIDConflictPolicy @@ -89,24 +89,22 @@ def producer(name: str, epoch: int) -> Iterator[None]: _producer_override.reset(token) -def _client_epoch(attempt: int) -> stream_v1.ProducerEpoch: - return stream_v1.ProducerEpoch(client=stream_v1.ClientEpoch(attempt=attempt)) - - def _client_producer( client: temporalio.client.Client, named: tuple[str, int] | None ) -> stream_v1.Producer: if named is not None: name, epoch = named return stream_v1.Producer( - id=_ids.named_client_producer_id(name), epoch=_client_epoch(epoch) + id=_ids.named_client_producer_id(name), + client=stream_v1.ClientProducer(epoch=stream_v1.ClientEpoch(attempt=epoch)), ) key = _CLIENT_KEYS.get(client) if key is None: key = f"{_process_key}-{next(_clients)}" _CLIENT_KEYS[client] = key return stream_v1.Producer( - id=_ids.client_producer_id(client.identity, key), epoch=_client_epoch(1) + id=_ids.client_producer_id(client.identity, key), + client=stream_v1.ClientProducer(epoch=stream_v1.ClientEpoch(attempt=1)), ) @@ -254,20 +252,10 @@ async def get_offset( with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): return StreamOffset._from_wire(await self._access().get_offset()) - def _check_close(self) -> None: - if self._ref.owner is not None: - # Only the run chain closes a Stream bound to a Workflow. The Store refuses a - # client's close too, but this check makes no call. - raise _errors.not_owner( - f"Can't close the Stream {self._ref.id} from a client, since it is bound to a " - "Workflow" - ) - async def close( self, rpc_metadata: Mapping[str, str | bytes], rpc_timeout: timedelta | None ) -> None: """Write the Records of these calls, then close the Stream.""" - self._check_close() with _backend.using(_backend.options_of(rpc_metadata, rpc_timeout)): await self._access().close() # Only a close that the Store took counts, so the end of async with tries a failed one @@ -277,9 +265,7 @@ async def close( self._raise_lost_write() async def enter(self) -> None: - """Check that the client can close the Stream at the end of the block.""" - # Fail early, since the close at the end would raise and hide the block's own error. - self._check_close() + """Enter the block that closes the Stream at its end.""" async def exit(self, exc: BaseException | None) -> None: """Close the Stream, unless these calls closed it already.""" @@ -324,7 +310,6 @@ def get_stream( payload_type: type[T] | None, *, id: str, - owner: models.Execution | None, backend: str | None, ) -> Stream[T]: """Get a Stream by its id. @@ -335,7 +320,7 @@ def get_stream( # An empty id would fail only at the first call, with an unclear error. raise ValueError("The Stream id is empty") backend_name, calls = client_backend(client, backend) - ref: StreamRef[T] = StreamRef(id=id, owner=owner, backend=backend_name) + ref: StreamRef[T] = StreamRef(id=id, backend=backend_name) return _stream(client, calls, ref, payload_type) diff --git a/temporalio/streams/_workflow.py b/temporalio/streams/_workflow.py index c45293e72b..4ab3e377d3 100644 --- a/temporalio/streams/_workflow.py +++ b/temporalio/streams/_workflow.py @@ -101,8 +101,6 @@ class _RunState: tails: dict[str, asyncio.Task[None]] = field(default_factory=dict) # The last write of each Stream that hasn't started, which later appends can join. waiting: dict[str, _Write] = field(default_factory=dict) - # The Streams on which a retry run's first append withdrew the previous run's Records. - withdrawn: set[str] = field(default_factory=set) # Whether the run created, bound or received a Stream. Only then do its Activities get a # Producer key, so a run with no Stream adds nothing to History. uses_streams: bool = False @@ -224,16 +222,6 @@ def _write( if waiting is not None and waiting.take(request): waiting.calls.append((_records(waiting.request), future)) return future - if isinstance(request, stream_pb.AppendRecordsRequest) and ( - stream_id not in state.withdrawn - ): - previous_run = _support.withdraw_run() - if previous_run: - # A retry run's first append to a Stream withdraws the Records of the run it - # retries. The Store refuses a run withdrawal on a create with no owner, and a - # Workflow's Stream has none, so only an append sends it. - request.writer.withdraw_run = previous_run - state.withdrawn.add(stream_id) write = _Write(activity, request, [(_records(request), future)]) if isinstance(request, stream_pb.AppendRecordsRequest): write.size = sum(record.ByteSize() for record in request.records) @@ -389,7 +377,7 @@ def _payload(value: Any) -> Any: def _model_ref(ref: StreamRef[Any]) -> models.StreamRef: - return models.StreamRef(id=ref.id, owner=ref.owner) + return models.StreamRef(id=ref.id) def _proto_ref(ref: StreamRef[Any]) -> stream_v1.StreamRef: @@ -397,14 +385,12 @@ def _proto_ref(ref: StreamRef[Any]) -> stream_v1.StreamRef: def _generated_ref(ref: StreamRef[Any]) -> models.StreamRef: - return models.StreamRef(id=ref.id, owner=ref.owner, backend=ref.backend or None) + return models.StreamRef(id=ref.id, backend=ref.backend or None) def _generated(ref: StreamRef[Any]) -> generated_stream.Stream[Any]: # The generated factory takes the ref's fields. - return generated_stream.get_stream( - id=ref.id, owner=ref.owner, backend=ref.backend or None - ) + return generated_stream.get_stream(id=ref.id, backend=ref.backend or None) def _check( @@ -459,28 +445,7 @@ def _start_close(ref: StreamRef[Any]) -> asyncio.Future[StreamOffset]: return _call_future(call, lambda _: 0) -def check_owner(ref: StreamRef[Any]) -> None: - """Check that this run chain may close the Stream. - - A Workflow closes a standalone Stream. Only its own run chain closes a bound Stream. - """ - owner = ref.owner - if owner is None: - return - info = temporalio.workflow.info() - if (owner.business_id, owner.run_id) != ( - info.workflow_id, - info.first_execution_run_id, - ): - # The backend refuses the close too, but this check fails before the Workflow - # schedules a command. - raise _errors.not_owner( - f"Can't close the Stream {ref.id}, since it is bound to another run chain" - ) - - def _closing(ref: StreamRef[Any]) -> _RunStream: - check_owner(ref) return _check(ref, "close") @@ -719,9 +684,6 @@ def _start_create( _check(ref, "create", backend_checked=True) proto = models._CreateStreamRequestTransferTypeConverter().to_transfer_type(request) proto.stream_ref.CopyFrom(_proto_ref(ref)) - # The Store refuses a run withdrawal on a create with no owner, and a Workflow's Stream has - # none. The run's first append withdraws the previous run instead. - proto.ClearField("withdraw_run") # A later run of the chain repeats the same create, so the id is the same in each run. The # Store then gives a retry run or a cron run the Stream that the first run made. info = temporalio.workflow.info() @@ -754,7 +716,7 @@ def operation_token(self) -> str | None: def _from_model(ref: models.StreamRef, backend: str | None) -> StreamRef[Any]: - return StreamRef(id=ref.id, owner=ref.owner, backend=backend) + return StreamRef(id=ref.id, backend=backend) # The Worker's interceptor runs the Stream calls of the generated Workflow API with these. They diff --git a/temporalio/streams/_writer.py b/temporalio/streams/_writer.py index b7f68c680b..e7ffb6f0cc 100644 --- a/temporalio/streams/_writer.py +++ b/temporalio/streams/_writer.py @@ -8,7 +8,6 @@ from typing import TYPE_CHECKING, Any import temporalio.api.stream.v1 as stream_v1 -from temporalio.api.enums.v1 import StreamErrorReason from temporalio.streams import _errors from temporalio.streams._offset import StreamOffset @@ -68,8 +67,6 @@ def __init__( stream_id: str, producer: stream_v1.Producer, free: Callable[[int], None] | None = None, - previous_completed_task: stream_v1.WorkflowTaskEpoch | None = None, - failed_tasks: Sequence[int] = (), ) -> None: """Create a writer. It opens on the first write. @@ -81,8 +78,6 @@ def __init__( self._producer = producer self._free = free self._release: weakref.finalize[Any, RecordWriter] | None = None - self._previous_completed_task = previous_completed_task - self._failed_tasks = list(failed_tasks) self._writer: Writer | None = None self._open_lock = asyncio.Lock() # The last write that started. Each write waits for the one before it, so the order is @@ -97,28 +92,8 @@ def producer(self) -> stream_v1.Producer: """The Producer that the writer writes as.""" return self._producer - async def withdraw_earlier_attempts(self) -> None: - """Open the writer of a new attempt, which withdraws the Records of earlier attempts. - - A Stream that doesn't exist has nothing to withdraw. A closed or expired Stream accepts - no write, so the attempt continues, and a later write raises the real error. - """ - try: - await self.open() - except _errors.StreamError as err: - if err.reason not in ( - StreamErrorReason.STREAM_ERROR_REASON_NOT_FOUND, - StreamErrorReason.STREAM_ERROR_REASON_CLOSED, - StreamErrorReason.STREAM_ERROR_REASON_EXPIRED, - ): - raise - async def open(self) -> Writer: - """Open the crate's writer once. - - The open of a new attempt withdraws the Records of earlier attempts, even if it writes - nothing. - """ + """Open the crate's writer once.""" from temporalio.streams.writer_system import models async with self._open_lock: @@ -126,18 +101,9 @@ async def open(self) -> Writer: producer = models._ProducerTransferTypeConverter().from_transfer_type( self._producer, models.Producer ) - previous = self._previous_completed_task self._writer = await self._writers.open_writer( ref=models.StreamRef(id=self._stream_id), - writer=models.StreamWriter( - producer=producer, - previous_completed_task=models.WorkflowTaskEpoch( - started_event_id=previous.started_event_id - ) - if previous is not None - else None, - failed_tasks=self._failed_tasks, - ), + writer=models.StreamWriter(producer=producer), ) if self._free is not None: # The crate keeps a writer for each open that nobody closed or freed. So diff --git a/temporalio/streams/_writes.py b/temporalio/streams/_writes.py index 153e0d88be..b024929f91 100644 --- a/temporalio/streams/_writes.py +++ b/temporalio/streams/_writes.py @@ -34,47 +34,28 @@ def _context() -> tuple[_activity.ActivityStreams, _backend.StreamServiceCalls]: def _producer(streams: _activity.ActivityStreams) -> models.Producer: # All attempts of one Workflow Task write with one epoch and sequence, so the Store dedupes a - # retry. A later task's higher epoch withdraws a failed task's Records for the same call key. - producer = streams.producer - local = producer.epoch.local_activity - started = local.task.started_time if local.task.HasField("started_time") else None + # retry. A later task's higher epoch supersedes a failed task's Records for the same call key. + local = streams.producer.local_activity + task_pb = local.epoch.task + started = task_pb.started_time if task_pb.HasField("started_time") else None task = models.WorkflowTaskEpoch( - started_event_id=local.task.started_event_id or None, - attempt=local.task.attempt or None, + started_event_id=task_pb.started_event_id or None, + attempt=task_pb.attempt or None, started_time=models.Timestamp(seconds=started.seconds, nanos=started.nanos) if started is not None else None, ) return models.Producer( - id=producer.id, - run_id=producer.run_id or None, - epoch=models.ProducerEpoch( - kind=models.ProducerEpochKindLocalActivity( - models.LocalActivityEpoch(task=task, attempt=1) + id=streams.producer.id, + kind=models.ProducerKindLocalActivity( + models.LocalActivityProducer( + run_id=local.run_id or None, + epoch=models.LocalActivityEpoch(task=task, attempt=1), ) ), ) -def _writer( - streams: _activity.ActivityStreams, - producer: models.Producer, - withdraw_run: str | None, -) -> models.StreamWriter: - # The run's task history, so the write withdraws the run's orphaned Local Activity Records. - previous = streams.previous_completed_task - return models.StreamWriter( - producer=producer, - previous_completed_task=models.WorkflowTaskEpoch( - started_event_id=previous.started_event_id - ) - if previous is not None - else None, - failed_tasks=list(streams.failed_tasks), - withdraw_run=withdraw_run, - ) - - def _model_ref(ref: stream_v1.StreamRef) -> models.StreamRef: return models._StreamRefTransferTypeConverter().from_transfer_type( ref, models.StreamRef @@ -194,7 +175,7 @@ async def append_records( producer = _producer(streams) producer_pb = models._ProducerTransferTypeConverter().to_transfer_type(producer) key = (streams.namespace, request.ref.id, producer_pb.id) - task = producer_pb.epoch.local_activity.task + task = producer_pb.local_activity.epoch.task epoch = ( task.started_event_id, task.attempt, @@ -217,13 +198,7 @@ async def append_records( response = await service.send_append_records( models.AppendRecordsRequest( ref=_model_ref(request.ref), - writer=_writer( - streams, - producer, - (request.writer.withdraw_run or None) - if index == 0 - else None, - ), + writer=models.StreamWriter(producer=producer), sequence=index + 1, records=[ models.AppendRecord( @@ -255,19 +230,13 @@ async def append_records( async def close_stream(request: stream_pb.CloseStreamRequest) -> int: """Close a Stream as the Workflow run that scheduled this Activity.""" streams, service = _context() - owner = request.ref.owner if request.ref.HasField("owner") else None - if owner is not None and owner.business_id != streams.workflow_id: - # The Store refuses an Activity Producer's close, so the Activity closes as the run. It - # refuses a Stream bound to another Workflow, and Workflow code checks the run chain. - raise _errors.not_owner( - f"Can't close the Stream {request.ref.id} for another Workflow" - ) + # The close Activity runs in the Workflow, so its Producer carries the run id, and the Store + # closes the Stream as the run's Workflow Producer. await service.send_close_stream( models.CloseStreamRequest( ref=_model_ref(request.ref), writer=models.StreamWriter( - producer=_ids.workflow_producer(streams.producer.run_id), - failed_tasks=[], + producer=_ids.workflow_producer(streams.producer.activity.run_id) ), ) ) diff --git a/temporalio/streams/superseded.py b/temporalio/streams/superseded.py new file mode 100644 index 0000000000..987b8f001b --- /dev/null +++ b/temporalio/streams/superseded.py @@ -0,0 +1,150 @@ +"""Opt-in client-side supersession for Stream reads. + +Import it explicitly. It is not part of the base ``temporalio.streams`` surface:: + + from temporalio.streams.superseded import Superseded, with_superseded + +The base read gives ``Record | Closed`` only. :py:func:`with_superseded` wraps a reader and adds +:py:class:`Superseded` items, computed from the ``Producer`` epoch that each Record carries, with +no Store facts. It holds the read's view in memory, so it is not for very large Streams. + +Only a failed Activity or Standalone Activity attempt is superseded: a later Record with the same +Activity ``id`` and ``run_id`` and a strictly higher epoch withdraws the earlier attempt's +Records. A client never supersedes, since a newer client epoch only fences. A Workflow write is +not superseded from the epochs alone, because a run's ``Producer`` id is stable across its +Workflow Tasks and the Store keeps no facts to tell a failed task from a completed one. + +.. warning:: + This module is experimental. +""" + +from __future__ import annotations + +import enum +from collections.abc import AsyncGenerator +from dataclasses import dataclass, field +from typing import Any, Literal + +from temporalio.streams._items import Closed, Record +from temporalio.streams._offset import StreamOffset +from temporalio.streams._reader import Reader +from temporalio.streams.system import models + + +class SupersededKind(enum.Enum): + """The kind of a :py:class:`Superseded` item.""" + + SUPERSEDED = "superseded" + + +@dataclass(frozen=True) +class Superseded: + """An item that withdraws the Records of a failed Activity attempt. + + :py:func:`with_superseded` yields it when a later Record has the same Activity ``id`` and + ``run_id`` and a strictly higher epoch. A live view removes the Records in + :py:attr:`withdrawn_offsets`. A view that kept Records from an earlier read checks them with + :py:meth:`withdraws`. + + .. warning:: + This API is experimental. + """ + + kind: Literal[SupersededKind.SUPERSEDED] = field( + default=SupersededKind.SUPERSEDED, init=False + ) + """Always :py:attr:`SupersededKind.SUPERSEDED`.""" + + offset: StreamOffset + """The offset of the Record whose higher epoch withdrew the earlier ones.""" + + producer: models.Producer | None + """The Producer whose higher epoch withdraws the earlier Records.""" + + withdrawn_offsets: tuple[StreamOffset, ...] = () + """The offsets of the Records that this read gave and this item withdraws, in order.""" + + def withdraws(self, record: Record[Any] | models.Producer | None) -> bool: + """Whether this item withdraws a Record, or the Records of a Producer. + + It answers for any earlier Record, such as one that an app showed before a resume, from + the Producer that each Record carries. + """ + other = record.producer if isinstance(record, Record) else record + if other is None or self.producer is None: + return False + group = _group(other) + if group is None or group != _group(self.producer): + return False + return _epoch_key(other) < _epoch_key(self.producer) + + +async def with_superseded( + reader: Reader[Any], +) -> AsyncGenerator[Record[Any] | Superseded | Closed, None]: + """Wrap a reader so it also yields :py:class:`Superseded` items. + + It yields each :py:class:`Record` and the :py:class:`Closed` item of the base read, and a + ``Superseded`` item before the Record whose higher Activity epoch withdraws the earlier + attempt's Records. + """ + # The offsets of the Records shown for each Activity Producer group, at its highest epoch so + # far. A higher epoch supersedes them. + shown: dict[tuple[str, str, str], tuple[tuple[int, ...], list[StreamOffset]]] = {} + try: + async for item in reader: + if isinstance(item, Record) and item.producer is not None: + group = _group(item.producer) + if group is not None: + key = _epoch_key(item.producer) + seen = shown.get(group) + if seen is not None and key > seen[0]: + yield Superseded( + offset=item.offset, + producer=item.producer, + withdrawn_offsets=tuple(seen[1]), + ) + shown[group] = (key, [item.offset]) + elif seen is not None and key == seen[0]: + seen[1].append(item.offset) + elif seen is None: + shown[group] = (key, [item.offset]) + yield item + finally: + await reader.aclose() + + +def _group(producer: models.Producer) -> tuple[str, str, str] | None: + # Only an Activity or Standalone Activity Producer supersedes. The group is its kind, id and + # run, so a retry of the same Activity supersedes its earlier attempt. + match producer.kind: + case models.ProducerKindActivity(value=activity): + return ("activity", producer.id, activity.run_id or "") + case models.ProducerKindStandaloneActivity(value=activity): + return ("standalone-activity", producer.id, activity.run_id or "") + case _: + return None + + +def _epoch_key(producer: models.Producer) -> tuple[int, ...]: + # An Activity epoch orders by scheduled time, then attempt. A reset starts the attempt count + # again, so the time orders the attempts across it. + match producer.kind: + case ( + models.ProducerKindActivity(value=activity) + | models.ProducerKindStandaloneActivity(value=activity) + ): + epoch = activity.epoch + scheduled = epoch.attempt_scheduled_time if epoch else None + attempt = epoch.attempt if epoch else None + return ( + _n(scheduled.seconds if scheduled else None), + _n(scheduled.nanos if scheduled else None), + _n(attempt), + ) + case _: + return (0,) + + +def _n(value: int | None) -> int: + return max(value or 0, 0) diff --git a/temporalio/streams/system/__init__.py b/temporalio/streams/system/__init__.py index c0d5432164..57cbb514c2 100644 --- a/temporalio/streams/system/__init__.py +++ b/temporalio/streams/system/__init__.py @@ -4,82 +4,86 @@ from .models import ( ActivityEpoch, + ActivityProducer, AppendRecord, AppendRecordsRequest, AppendRecordsResponse, ClientEpoch, + ClientProducer, CloseStreamRequest, CloseStreamResponse, CreateStreamRequest, CreateStreamResult, DescribeStreamRequest, DescribeStreamResponse, - Execution, - ExecutionType, LocalActivityEpoch, + LocalActivityProducer, Producer, - ProducerEpoch, - ProducerEpochKind, - ProducerEpochKindActivity, - ProducerEpochKindClient, - ProducerEpochKindLocalActivity, - ProducerEpochKindStandaloneActivity, - ProducerEpochKindWorkflow, + ProducerKind, + ProducerKindActivity, + ProducerKindClient, + ProducerKindLocalActivity, + ProducerKindStandaloneActivity, + ProducerKindWorkflow, ReadRecordsRequest, ReadRecordsResponse, + StreamClose, + StreamCloseStatus, StreamData, StreamIdConflictPolicy, StreamItem, + StreamItemClose, StreamItemData, - StreamItemSuperseded, StreamReadStart, StreamRecord, StreamRef, StreamStatus, - StreamSuperseded, StreamWriter, Timestamp, + WorkflowProducer, WorkflowTaskEpoch, ) from .services import StreamService __all__ = [ "ActivityEpoch", + "ActivityProducer", "AppendRecord", "AppendRecordsRequest", "AppendRecordsResponse", "ClientEpoch", + "ClientProducer", "CloseStreamRequest", "CloseStreamResponse", "CreateStreamRequest", "CreateStreamResult", "DescribeStreamRequest", "DescribeStreamResponse", - "Execution", - "ExecutionType", "LocalActivityEpoch", + "LocalActivityProducer", "Producer", - "ProducerEpoch", - "ProducerEpochKind", - "ProducerEpochKindWorkflow", - "ProducerEpochKindLocalActivity", - "ProducerEpochKindActivity", - "ProducerEpochKindStandaloneActivity", - "ProducerEpochKindClient", + "ProducerKind", + "ProducerKindWorkflow", + "ProducerKindLocalActivity", + "ProducerKindActivity", + "ProducerKindStandaloneActivity", + "ProducerKindClient", "ReadRecordsRequest", "ReadRecordsResponse", + "StreamClose", + "StreamCloseStatus", "StreamData", "StreamIdConflictPolicy", "StreamItem", "StreamItemData", - "StreamItemSuperseded", + "StreamItemClose", "StreamReadStart", "StreamRecord", "StreamRef", "StreamStatus", - "StreamSuperseded", "StreamWriter", "Timestamp", + "WorkflowProducer", "WorkflowTaskEpoch", "StreamService", ] diff --git a/temporalio/streams/system/client/_extensions.py b/temporalio/streams/system/client/_extensions.py index 62d8001ef8..c4e8e8035d 100644 --- a/temporalio/streams/system/client/_extensions.py +++ b/temporalio/streams/system/client/_extensions.py @@ -15,7 +15,6 @@ from .._resource_call import resource_call from ..models import ( CreateStreamRequest, - Execution, StreamIdConflictPolicy, StreamRef, ) @@ -128,7 +127,6 @@ async def create_stream( self, *, id: str, - owner: Execution | None = None, backend: str | None = None, retention: datetime.timedelta | None = None, id_conflict_policy: StreamIdConflictPolicy | None = None, @@ -143,7 +141,6 @@ async def create_stream( self, *, id: str, - owner: Execution | None = None, backend: str | None = None, retention: datetime.timedelta | None = None, id_conflict_policy: StreamIdConflictPolicy | None = None, @@ -157,7 +154,6 @@ async def create_stream( self, *, id: str, - owner: Execution | None = None, backend: str | None = None, retention: datetime.timedelta | None = None, id_conflict_policy: StreamIdConflictPolicy | None = None, @@ -168,8 +164,6 @@ async def create_stream( """ Args: id: The Stream id, 1 to 4096 bytes. - owner: The run chain or Standalone Activity of a bound Stream. Empty if - standalone. backend: Registered backend name, empty for the default. The Store never reads it. retention: The age after which a Record is dropped. Empty is the Store @@ -181,13 +175,11 @@ async def create_stream( body_type = typing.Any stream_ref = StreamRef( id=id, - owner=owner, backend=backend, ) request = CreateStreamRequest( stream_ref=stream_ref, retention=retention, - withdraw_run=_support.withdraw_run(), id_conflict_policy=id_conflict_policy, request_id=_support.request_id(), ) diff --git a/temporalio/streams/system/client/_resources/stream.py b/temporalio/streams/system/client/_resources/stream.py index fd5d4c8101..46bca73d58 100644 --- a/temporalio/streams/system/client/_resources/stream.py +++ b/temporalio/streams/system/client/_resources/stream.py @@ -68,7 +68,6 @@ def ref(self) -> StreamRef[BodyT]: if not typing.get_args(getattr(self._ref, "__orig_class__", None)): self._ref = StreamRef[self.body_type]( id=self._ref.id, - owner=self._ref.owner, backend=self._ref.backend, ) return self._ref @@ -179,8 +178,7 @@ async def iter_read( 1 fails. start: Where to start when from-offset is empty. Unspecified is the beginning. - max_records: About this many items per page, 1 to 1000. A Superseded item - stays with its Record. + max_records: About this many items per page, 1 to 1000. wait: How long to wait for a new Record before an empty answer. At most 60 s. create_wait: How long to wait for an uncreated Stream. Empty fails at once. @@ -238,8 +236,7 @@ def iter_read_items( 1 fails. start: Where to start when from-offset is empty. Unspecified is the beginning. - max_records: About this many items per page, 1 to 1000. A Superseded item - stays with its Record. + max_records: About this many items per page, 1 to 1000. wait: How long to wait for a new Record before an empty answer. At most 60 s. create_wait: How long to wait for an uncreated Stream. Empty fails at once. diff --git a/temporalio/streams/system/models.py b/temporalio/streams/system/models.py index b89b436144..6e5cac8e08 100644 --- a/temporalio/streams/system/models.py +++ b/temporalio/streams/system/models.py @@ -10,8 +10,6 @@ import google.protobuf.timestamp_pb2 import typing_extensions -import temporalio.api.common.v1.message_pb2 -import temporalio.api.enums.v1.common_pb2 import temporalio.api.enums.v1.stream_pb2 import temporalio.api.stream.v1.message_pb2 import temporalio.api.streamservice.v1.request_response_pb2 @@ -19,21 +17,6 @@ import temporalio.nexus.system._support as _support -class ExecutionType(enum.IntEnum): - ExecutionTypeUnspecified = 0 - ExecutionTypeWorkflow = 1 - ExecutionTypeActivity = 2 - ExecutionTypeNexusOperation = 3 - - -def _execution_type_from_proto(value: int) -> ExecutionType: - try: - return ExecutionType(value) - except ValueError: - # A newer peer can send a case that this enum doesn't know. - return ExecutionType.ExecutionTypeUnspecified - - class StreamIdConflictPolicy(enum.IntEnum): StreamIdConflictPolicyUnspecified = 0 StreamIdConflictPolicyFail = 1 @@ -46,6 +29,25 @@ class StreamReadStart(enum.IntEnum): StreamReadStartEnd = 2 +class StreamCloseStatus(enum.IntEnum): + StreamCloseStatusUnspecified = 0 + StreamCloseStatusClosed = 1 + StreamCloseStatusOwnerCompleted = 2 + StreamCloseStatusOwnerFailed = 3 + StreamCloseStatusOwnerCanceled = 4 + StreamCloseStatusOwnerTerminated = 5 + StreamCloseStatusOwnerTimedOut = 6 + StreamCloseStatusIdle = 7 + + +def _stream_close_status_from_proto(value: int) -> StreamCloseStatus: + try: + return StreamCloseStatus(value) + except ValueError: + # A newer peer can send a case that this enum doesn't know. + return StreamCloseStatus.StreamCloseStatusUnspecified + + class StreamStatus(enum.IntEnum): StreamStatusUnspecified = 0 StreamStatusOpen = 1 @@ -71,7 +73,6 @@ class CreateStreamRequest: stream_ref: StreamRef retention: datetime.timedelta | None = None - withdraw_run: str | None = None id_conflict_policy: StreamIdConflictPolicy | None = None request_id: str | None = None @@ -103,7 +104,6 @@ def from_transfer_type( retention=_support.duration_from_proto(value.retention) if value.HasField("retention") else None, - withdraw_run=value.withdraw_run if bool(value.withdraw_run) else None, id_conflict_policy=StreamIdConflictPolicy(value.id_conflict_policy) if value.id_conflict_policy != 0 else None, @@ -123,8 +123,6 @@ def to_transfer_type( ) if value.retention is not None: message.retention.CopyFrom(_support.duration_to_proto(value.retention)) - if value.withdraw_run is not None: - message.withdraw_run = value.withdraw_run if value.id_conflict_policy is not None: message.id_conflict_policy = ( temporalio.api.enums.v1.stream_pb2.StreamIdConflictPolicy.ValueType( @@ -146,7 +144,6 @@ class StreamRef(typing.Generic[_BodyT]): """Identifies one Stream. The id is opaque to callers.""" id: str - owner: Execution | None = None backend: str | None = None @@ -170,11 +167,6 @@ def from_transfer_type( id = value.id return type_hint( id=id, - owner=_ExecutionTransferTypeConverter().from_transfer_type( - value.owner, Execution - ) - if value.HasField("owner") - else None, backend=value.backend if bool(value.backend) else None, ) @@ -185,10 +177,6 @@ def to_transfer_type( ) -> temporalio.api.stream.v1.message_pb2.StreamRef: message = temporalio.api.stream.v1.message_pb2.StreamRef() message.id = value.id - if value.owner is not None: - message.owner.CopyFrom( - _ExecutionTransferTypeConverter().to_transfer_type(value.owner) - ) if value.backend is not None: message.backend = value.backend return message @@ -199,58 +187,6 @@ def to_transfer_type( ) # pyright: ignore[reportUnusedCallResult] -@dataclasses.dataclass(slots=True) -class Execution: - """A Workflow run chain or a Standalone Activity run.""" - - type: ExecutionType | None = None - business_id: str | None = None - run_id: str | None = None - - -class _ExecutionTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - Execution, temporalio.api.common.v1.message_pb2.Execution - ] -): - transfer_type: type[temporalio.api.common.v1.message_pb2.Execution] | None = ( - temporalio.api.common.v1.message_pb2.Execution - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.common.v1.message_pb2.Execution, - type_hint: type[Execution], - ) -> Execution: - return Execution( - type=_execution_type_from_proto(value.type) if value.type != 0 else None, - business_id=value.business_id if bool(value.business_id) else None, - run_id=value.run_id if bool(value.run_id) else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: Execution, - ) -> temporalio.api.common.v1.message_pb2.Execution: - message = temporalio.api.common.v1.message_pb2.Execution() - if value.type is not None: - message.type = temporalio.api.enums.v1.common_pb2.ExecutionType.ValueType( - value.type - ) - if value.business_id is not None: - message.business_id = value.business_id - if value.run_id is not None: - message.run_id = value.run_id - return message - - -temporalio.converter.transfer_type_convertible(_ExecutionTransferTypeConverter)( - Execution -) # pyright: ignore[reportUnusedCallResult] - - @dataclasses.dataclass(slots=True) class CreateStreamResult: ref: StreamRef | None = None @@ -433,14 +369,11 @@ def to_transfer_type( )(AppendRecord) # pyright: ignore[reportUnusedCallResult] -@dataclasses.dataclass(slots=True, kw_only=True) +@dataclasses.dataclass(slots=True) class StreamWriter: - """The Producer of a write, and which earlier Records the write withdraws.""" + """The Producer of a write. An append and a close carry the same writer.""" producer: Producer | None = None - previous_completed_task: WorkflowTaskEpoch | None = None - failed_tasks: list[int] - withdraw_run: str | None = None class _StreamWriterTransferTypeConverter( @@ -458,20 +391,12 @@ def from_transfer_type( value: temporalio.api.stream.v1.message_pb2.StreamWriter, type_hint: type[StreamWriter], ) -> StreamWriter: - failed_tasks = list(value.failed_tasks) return StreamWriter( producer=_ProducerTransferTypeConverter().from_transfer_type( value.producer, Producer ) if value.HasField("producer") else None, - previous_completed_task=_WorkflowTaskEpochTransferTypeConverter().from_transfer_type( - value.previous_completed_task, WorkflowTaskEpoch - ) - if value.HasField("previous_completed_task") - else None, - failed_tasks=failed_tasks, - withdraw_run=value.withdraw_run if bool(value.withdraw_run) else None, ) @typing_extensions.override @@ -484,15 +409,6 @@ def to_transfer_type( message.producer.CopyFrom( _ProducerTransferTypeConverter().to_transfer_type(value.producer) ) - if value.previous_completed_task is not None: - message.previous_completed_task.CopyFrom( - _WorkflowTaskEpochTransferTypeConverter().to_transfer_type( - value.previous_completed_task - ) - ) - message.failed_tasks.extend(value.failed_tasks) - if value.withdraw_run is not None: - message.withdraw_run = value.withdraw_run return message @@ -503,11 +419,10 @@ def to_transfer_type( @dataclasses.dataclass(slots=True) class Producer: - """The Producer of a batch.""" + """The code that wrote a batch. Its id is stable across attempts.""" id: str - epoch: ProducerEpoch | None = None - run_id: str | None = None + kind: ProducerKind | None = None class _ProducerTransferTypeConverter( @@ -528,145 +443,146 @@ def from_transfer_type( if not value.id: raise ValueError("missing required field Producer.id") id = value.id - return Producer( - id=id, - epoch=_ProducerEpochTransferTypeConverter().from_transfer_type( - value.epoch, ProducerEpoch - ) - if value.HasField("epoch") - else None, - run_id=value.run_id if bool(value.run_id) else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: Producer, - ) -> temporalio.api.stream.v1.message_pb2.Producer: - message = temporalio.api.stream.v1.message_pb2.Producer() - message.id = value.id - if value.epoch is not None: - message.epoch.CopyFrom( - _ProducerEpochTransferTypeConverter().to_transfer_type(value.epoch) - ) - if value.run_id is not None: - message.run_id = value.run_id - return message - - -temporalio.converter.transfer_type_convertible(_ProducerTransferTypeConverter)(Producer) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class ProducerEpoch: - """The ordered attempt of a Producer. A higher epoch supersedes a lower one.""" - - kind: ProducerEpochKind | None = None - - -class _ProducerEpochTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - ProducerEpoch, temporalio.api.stream.v1.message_pb2.ProducerEpoch - ] -): - transfer_type: type[temporalio.api.stream.v1.message_pb2.ProducerEpoch] | None = ( - temporalio.api.stream.v1.message_pb2.ProducerEpoch - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.ProducerEpoch, - type_hint: type[ProducerEpoch], - ) -> ProducerEpoch: _oneof_kind: ( - ProducerEpochKindWorkflow - | ProducerEpochKindLocalActivity - | ProducerEpochKindActivity - | ProducerEpochKindStandaloneActivity - | ProducerEpochKindClient + ProducerKindWorkflow + | ProducerKindLocalActivity + | ProducerKindActivity + | ProducerKindStandaloneActivity + | ProducerKindClient | None ) match value.WhichOneof("kind"): case "workflow": - _oneof_kind = ProducerEpochKindWorkflow( - _WorkflowTaskEpochTransferTypeConverter().from_transfer_type( - value.workflow, WorkflowTaskEpoch + _oneof_kind = ProducerKindWorkflow( + _WorkflowProducerTransferTypeConverter().from_transfer_type( + value.workflow, WorkflowProducer ) ) case "local_activity": - _oneof_kind = ProducerEpochKindLocalActivity( - _LocalActivityEpochTransferTypeConverter().from_transfer_type( - value.local_activity, LocalActivityEpoch + _oneof_kind = ProducerKindLocalActivity( + _LocalActivityProducerTransferTypeConverter().from_transfer_type( + value.local_activity, LocalActivityProducer ) ) case "activity": - _oneof_kind = ProducerEpochKindActivity( - _ActivityEpochTransferTypeConverter().from_transfer_type( - value.activity, ActivityEpoch + _oneof_kind = ProducerKindActivity( + _ActivityProducerTransferTypeConverter().from_transfer_type( + value.activity, ActivityProducer ) ) case "standalone_activity": - _oneof_kind = ProducerEpochKindStandaloneActivity( - _ActivityEpochTransferTypeConverter().from_transfer_type( - value.standalone_activity, ActivityEpoch + _oneof_kind = ProducerKindStandaloneActivity( + _ActivityProducerTransferTypeConverter().from_transfer_type( + value.standalone_activity, ActivityProducer ) ) case "client": - _oneof_kind = ProducerEpochKindClient( - _ClientEpochTransferTypeConverter().from_transfer_type( - value.client, ClientEpoch + _oneof_kind = ProducerKindClient( + _ClientProducerTransferTypeConverter().from_transfer_type( + value.client, ClientProducer ) ) case None: _oneof_kind = None - return ProducerEpoch( + return Producer( + id=id, kind=_oneof_kind, ) @typing_extensions.override def to_transfer_type( self, - value: ProducerEpoch, - ) -> temporalio.api.stream.v1.message_pb2.ProducerEpoch: - message = temporalio.api.stream.v1.message_pb2.ProducerEpoch() + value: Producer, + ) -> temporalio.api.stream.v1.message_pb2.Producer: + message = temporalio.api.stream.v1.message_pb2.Producer() + message.id = value.id if value.kind is not None: match value.kind: - case ProducerEpochKindWorkflow(): + case ProducerKindWorkflow(): message.workflow.CopyFrom( - _WorkflowTaskEpochTransferTypeConverter().to_transfer_type( + _WorkflowProducerTransferTypeConverter().to_transfer_type( value.kind.value ) ) - case ProducerEpochKindLocalActivity(): + case ProducerKindLocalActivity(): message.local_activity.CopyFrom( - _LocalActivityEpochTransferTypeConverter().to_transfer_type( + _LocalActivityProducerTransferTypeConverter().to_transfer_type( value.kind.value ) ) - case ProducerEpochKindActivity(): + case ProducerKindActivity(): message.activity.CopyFrom( - _ActivityEpochTransferTypeConverter().to_transfer_type( + _ActivityProducerTransferTypeConverter().to_transfer_type( value.kind.value ) ) - case ProducerEpochKindStandaloneActivity(): + case ProducerKindStandaloneActivity(): message.standalone_activity.CopyFrom( - _ActivityEpochTransferTypeConverter().to_transfer_type( + _ActivityProducerTransferTypeConverter().to_transfer_type( value.kind.value ) ) - case ProducerEpochKindClient(): + case ProducerKindClient(): message.client.CopyFrom( - _ClientEpochTransferTypeConverter().to_transfer_type( + _ClientProducerTransferTypeConverter().to_transfer_type( value.kind.value ) ) return message -temporalio.converter.transfer_type_convertible(_ProducerEpochTransferTypeConverter)( - ProducerEpoch +temporalio.converter.transfer_type_convertible(_ProducerTransferTypeConverter)(Producer) # pyright: ignore[reportUnusedCallResult] + + +@dataclasses.dataclass(slots=True) +class WorkflowProducer: + """A Workflow run Producer. A higher epoch withdraws the run's orphaned Records.""" + + run_id: str | None = None + epoch: WorkflowTaskEpoch | None = None + + +class _WorkflowProducerTransferTypeConverter( + temporalio.converter.TransferTypeConverter[ + WorkflowProducer, temporalio.api.stream.v1.message_pb2.WorkflowProducer + ] +): + transfer_type: ( + type[temporalio.api.stream.v1.message_pb2.WorkflowProducer] | None + ) = temporalio.api.stream.v1.message_pb2.WorkflowProducer + + @typing_extensions.override + def from_transfer_type( + self, + value: temporalio.api.stream.v1.message_pb2.WorkflowProducer, + type_hint: type[WorkflowProducer], + ) -> WorkflowProducer: + return WorkflowProducer( + run_id=value.run_id if bool(value.run_id) else None, + epoch=_WorkflowTaskEpochTransferTypeConverter().from_transfer_type( + value.epoch, WorkflowTaskEpoch + ) + if value.HasField("epoch") + else None, + ) + + @typing_extensions.override + def to_transfer_type( + self, + value: WorkflowProducer, + ) -> temporalio.api.stream.v1.message_pb2.WorkflowProducer: + message = temporalio.api.stream.v1.message_pb2.WorkflowProducer() + if value.run_id is not None: + message.run_id = value.run_id + if value.epoch is not None: + message.epoch.CopyFrom( + _WorkflowTaskEpochTransferTypeConverter().to_transfer_type(value.epoch) + ) + return message + + +temporalio.converter.transfer_type_convertible(_WorkflowProducerTransferTypeConverter)( + WorkflowProducer ) # pyright: ignore[reportUnusedCallResult] @@ -772,6 +688,59 @@ def to_transfer_type( ) # pyright: ignore[reportUnusedCallResult] +@dataclasses.dataclass(slots=True) +class LocalActivityProducer: + """A Local Activity Producer. A higher epoch withdraws the Producer's lower epochs.""" + + run_id: str | None = None + epoch: LocalActivityEpoch | None = None + + +class _LocalActivityProducerTransferTypeConverter( + temporalio.converter.TransferTypeConverter[ + LocalActivityProducer, + temporalio.api.stream.v1.message_pb2.LocalActivityProducer, + ] +): + transfer_type: ( + type[temporalio.api.stream.v1.message_pb2.LocalActivityProducer] | None + ) = temporalio.api.stream.v1.message_pb2.LocalActivityProducer + + @typing_extensions.override + def from_transfer_type( + self, + value: temporalio.api.stream.v1.message_pb2.LocalActivityProducer, + type_hint: type[LocalActivityProducer], + ) -> LocalActivityProducer: + return LocalActivityProducer( + run_id=value.run_id if bool(value.run_id) else None, + epoch=_LocalActivityEpochTransferTypeConverter().from_transfer_type( + value.epoch, LocalActivityEpoch + ) + if value.HasField("epoch") + else None, + ) + + @typing_extensions.override + def to_transfer_type( + self, + value: LocalActivityProducer, + ) -> temporalio.api.stream.v1.message_pb2.LocalActivityProducer: + message = temporalio.api.stream.v1.message_pb2.LocalActivityProducer() + if value.run_id is not None: + message.run_id = value.run_id + if value.epoch is not None: + message.epoch.CopyFrom( + _LocalActivityEpochTransferTypeConverter().to_transfer_type(value.epoch) + ) + return message + + +temporalio.converter.transfer_type_convertible( + _LocalActivityProducerTransferTypeConverter +)(LocalActivityProducer) # pyright: ignore[reportUnusedCallResult] + + @dataclasses.dataclass(slots=True) class LocalActivityEpoch: """A Local Activity attempt. Its Workflow Task orders it first.""" @@ -824,6 +793,58 @@ def to_transfer_type( )(LocalActivityEpoch) # pyright: ignore[reportUnusedCallResult] +@dataclasses.dataclass(slots=True) +class ActivityProducer: + """An Activity Producer. A higher epoch withdraws the Producer's lower epochs.""" + + run_id: str | None = None + epoch: ActivityEpoch | None = None + + +class _ActivityProducerTransferTypeConverter( + temporalio.converter.TransferTypeConverter[ + ActivityProducer, temporalio.api.stream.v1.message_pb2.ActivityProducer + ] +): + transfer_type: ( + type[temporalio.api.stream.v1.message_pb2.ActivityProducer] | None + ) = temporalio.api.stream.v1.message_pb2.ActivityProducer + + @typing_extensions.override + def from_transfer_type( + self, + value: temporalio.api.stream.v1.message_pb2.ActivityProducer, + type_hint: type[ActivityProducer], + ) -> ActivityProducer: + return ActivityProducer( + run_id=value.run_id if bool(value.run_id) else None, + epoch=_ActivityEpochTransferTypeConverter().from_transfer_type( + value.epoch, ActivityEpoch + ) + if value.HasField("epoch") + else None, + ) + + @typing_extensions.override + def to_transfer_type( + self, + value: ActivityProducer, + ) -> temporalio.api.stream.v1.message_pb2.ActivityProducer: + message = temporalio.api.stream.v1.message_pb2.ActivityProducer() + if value.run_id is not None: + message.run_id = value.run_id + if value.epoch is not None: + message.epoch.CopyFrom( + _ActivityEpochTransferTypeConverter().to_transfer_type(value.epoch) + ) + return message + + +temporalio.converter.transfer_type_convertible(_ActivityProducerTransferTypeConverter)( + ActivityProducer +) # pyright: ignore[reportUnusedCallResult] + + @dataclasses.dataclass(slots=True) class ActivityEpoch: """An Activity or Standalone Activity attempt.""" @@ -878,6 +899,54 @@ def to_transfer_type( ) # pyright: ignore[reportUnusedCallResult] +@dataclasses.dataclass(slots=True) +class ClientProducer: + """A client Producer. A higher epoch only fences the lower ones.""" + + epoch: ClientEpoch | None = None + + +class _ClientProducerTransferTypeConverter( + temporalio.converter.TransferTypeConverter[ + ClientProducer, temporalio.api.stream.v1.message_pb2.ClientProducer + ] +): + transfer_type: type[temporalio.api.stream.v1.message_pb2.ClientProducer] | None = ( + temporalio.api.stream.v1.message_pb2.ClientProducer + ) + + @typing_extensions.override + def from_transfer_type( + self, + value: temporalio.api.stream.v1.message_pb2.ClientProducer, + type_hint: type[ClientProducer], + ) -> ClientProducer: + return ClientProducer( + epoch=_ClientEpochTransferTypeConverter().from_transfer_type( + value.epoch, ClientEpoch + ) + if value.HasField("epoch") + else None, + ) + + @typing_extensions.override + def to_transfer_type( + self, + value: ClientProducer, + ) -> temporalio.api.stream.v1.message_pb2.ClientProducer: + message = temporalio.api.stream.v1.message_pb2.ClientProducer() + if value.epoch is not None: + message.epoch.CopyFrom( + _ClientEpochTransferTypeConverter().to_transfer_type(value.epoch) + ) + return message + + +temporalio.converter.transfer_type_convertible(_ClientProducerTransferTypeConverter)( + ClientProducer +) # pyright: ignore[reportUnusedCallResult] + + @dataclasses.dataclass(slots=True) class ClientEpoch: """A client attempt, which the client picks.""" @@ -1243,7 +1312,7 @@ def from_transfer_type( type_hint: type[StreamRecord[BodyT]], ) -> StreamRecord[BodyT]: (body_type,) = typing.get_args(type_hint) or (typing.Any,) - _oneof_item: StreamItemData[BodyT] | StreamItemSuperseded | None + _oneof_item: StreamItemData[BodyT] | StreamItemClose | None match value.WhichOneof("item"): case "data": _oneof_item = StreamItemData( @@ -1254,10 +1323,10 @@ def from_transfer_type( ), ) ) - case "superseded": - _oneof_item = StreamItemSuperseded( - _StreamSupersededTransferTypeConverter().from_transfer_type( - value.superseded, StreamSuperseded + case "close": + _oneof_item = StreamItemClose( + _StreamCloseTransferTypeConverter().from_transfer_type( + value.close, StreamClose ) ) case None: @@ -1281,9 +1350,9 @@ def to_transfer_type( value.item.value ) ) - case StreamItemSuperseded(): - message.superseded.CopyFrom( - _StreamSupersededTransferTypeConverter().to_transfer_type( + case StreamItemClose(): + message.close.CopyFrom( + _StreamCloseTransferTypeConverter().to_transfer_type( value.item.value ) ) @@ -1353,50 +1422,72 @@ def to_transfer_type( @dataclasses.dataclass(slots=True) -class StreamSuperseded: - """Which earlier Records a Superseded item withdraws.""" +class StreamClose: + """The close of a Stream. A read gives it last.""" - writer: StreamWriter | None = None + producer: Producer | None = None + result: typing.Any | None = None + failure: BaseException | None = None + status: StreamCloseStatus | None = None -class _StreamSupersededTransferTypeConverter( +class _StreamCloseTransferTypeConverter( temporalio.converter.TransferTypeConverter[ - StreamSuperseded, temporalio.api.stream.v1.message_pb2.StreamSuperseded + StreamClose, temporalio.api.stream.v1.message_pb2.StreamClose ] ): - transfer_type: ( - type[temporalio.api.stream.v1.message_pb2.StreamSuperseded] | None - ) = temporalio.api.stream.v1.message_pb2.StreamSuperseded + transfer_type: type[temporalio.api.stream.v1.message_pb2.StreamClose] | None = ( + temporalio.api.stream.v1.message_pb2.StreamClose + ) @typing_extensions.override def from_transfer_type( self, - value: temporalio.api.stream.v1.message_pb2.StreamSuperseded, - type_hint: type[StreamSuperseded], - ) -> StreamSuperseded: - return StreamSuperseded( - writer=_StreamWriterTransferTypeConverter().from_transfer_type( - value.writer, StreamWriter + value: temporalio.api.stream.v1.message_pb2.StreamClose, + type_hint: type[StreamClose], + ) -> StreamClose: + return StreamClose( + producer=_ProducerTransferTypeConverter().from_transfer_type( + value.producer, Producer ) - if value.HasField("writer") + if value.HasField("producer") + else None, + result=_support.payload_from_proto(value.result) + if value.HasField("result") + else None, + failure=_support.failure_from_proto(value.failure) + if value.HasField("failure") + else None, + status=_stream_close_status_from_proto(value.status) + if value.status != 0 else None, ) @typing_extensions.override def to_transfer_type( self, - value: StreamSuperseded, - ) -> temporalio.api.stream.v1.message_pb2.StreamSuperseded: - message = temporalio.api.stream.v1.message_pb2.StreamSuperseded() - if value.writer is not None: - message.writer.CopyFrom( - _StreamWriterTransferTypeConverter().to_transfer_type(value.writer) + value: StreamClose, + ) -> temporalio.api.stream.v1.message_pb2.StreamClose: + message = temporalio.api.stream.v1.message_pb2.StreamClose() + if value.producer is not None: + message.producer.CopyFrom( + _ProducerTransferTypeConverter().to_transfer_type(value.producer) + ) + if value.result is not None: + message.result.CopyFrom(_support.payload_to_proto(value.result)) + if value.failure is not None: + message.failure.CopyFrom(_support.failure_to_proto(value.failure)) + if value.status is not None: + message.status = ( + temporalio.api.enums.v1.stream_pb2.StreamCloseStatus.ValueType( + value.status + ) ) return message -temporalio.converter.transfer_type_convertible(_StreamSupersededTransferTypeConverter)( - StreamSuperseded +temporalio.converter.transfer_type_convertible(_StreamCloseTransferTypeConverter)( + StreamClose ) # pyright: ignore[reportUnusedCallResult] @@ -1507,38 +1598,38 @@ def to_transfer_type( @dataclasses.dataclass(slots=True) -class ProducerEpochKindWorkflow: - value: WorkflowTaskEpoch +class ProducerKindWorkflow: + value: WorkflowProducer @dataclasses.dataclass(slots=True) -class ProducerEpochKindLocalActivity: - value: LocalActivityEpoch +class ProducerKindLocalActivity: + value: LocalActivityProducer @dataclasses.dataclass(slots=True) -class ProducerEpochKindActivity: - value: ActivityEpoch +class ProducerKindActivity: + value: ActivityProducer @dataclasses.dataclass(slots=True) -class ProducerEpochKindStandaloneActivity: - value: ActivityEpoch +class ProducerKindStandaloneActivity: + value: ActivityProducer @dataclasses.dataclass(slots=True) -class ProducerEpochKindClient: - value: ClientEpoch +class ProducerKindClient: + value: ClientProducer -ProducerEpochKind = ( - ProducerEpochKindWorkflow - | ProducerEpochKindLocalActivity - | ProducerEpochKindActivity - | ProducerEpochKindStandaloneActivity - | ProducerEpochKindClient +ProducerKind = ( + ProducerKindWorkflow + | ProducerKindLocalActivity + | ProducerKindActivity + | ProducerKindStandaloneActivity + | ProducerKindClient ) -"""The epoch of each kind of Producer.""" +"""The kind of a Producer, with its run and epoch.""" @dataclasses.dataclass(slots=True) @@ -1547,9 +1638,9 @@ class StreamItemData(typing.Generic[BodyT]): @dataclasses.dataclass(slots=True) -class StreamItemSuperseded: - value: StreamSuperseded +class StreamItemClose: + value: StreamClose -StreamItem = StreamItemData[BodyT] | StreamItemSuperseded -"""A read item: a Record or a Superseded item. A close writes no item.""" +StreamItem = StreamItemData[BodyT] | StreamItemClose +"""A read item: a Record or the close.""" diff --git a/temporalio/streams/system/workflow/_resources/stream.py b/temporalio/streams/system/workflow/_resources/stream.py index 7e24fd6bdd..b2edda1ef3 100644 --- a/temporalio/streams/system/workflow/_resources/stream.py +++ b/temporalio/streams/system/workflow/_resources/stream.py @@ -20,7 +20,6 @@ CloseStreamResponse, DescribeStreamRequest, DescribeStreamResponse, - Execution, ReadRecordsRequest, ReadRecordsResponse, StreamReadStart, @@ -63,7 +62,6 @@ def ref(self) -> StreamRef[BodyT]: if not typing.get_args(getattr(self._ref, "__orig_class__", None)): self._ref = StreamRef[self.body_type]( id=self._ref.id, - owner=self._ref.owner, backend=self._ref.backend, ) return self._ref @@ -157,7 +155,6 @@ def get_stream( def get_stream( *, id: str, - owner: Execution | None = None, backend: str | None = None, created: bool | None = None, body_type: type[BodyT], @@ -168,7 +165,6 @@ def get_stream( def get_stream( *, id: str, - owner: Execution | None = None, backend: str | None = None, created: bool | None = None, body_type: None = None, @@ -179,7 +175,6 @@ def get_stream( *, ref: StreamRef | None = None, id: str | None = None, - owner: Execution | None = None, backend: str | None = None, created: bool | None = None, body_type: typing.Any = None, @@ -187,7 +182,7 @@ def get_stream( """A handle on an existing ``Stream``, with no call.""" if ref is None: assert id is not None - ref = StreamRef(id=id, owner=owner, backend=backend) + ref = StreamRef(id=id, backend=backend) if body_type is None: body_type = typing.Any return Stream[body_type]( @@ -319,8 +314,7 @@ async def read_records( from_offset: Start offset. Empty uses start. An offset past the newest one + 1 fails. start: Where to start when from-offset is empty. Unspecified is the beginning. - max_records: About this many items per page, 1 to 1000. A Superseded item stays - with its Record. + max_records: About this many items per page, 1 to 1000. wait: How long to wait for a new Record before an empty answer. At most 60 s. create_wait: How long to wait for an uncreated Stream. Empty fails at once. At most 60 s. diff --git a/temporalio/streams/system/workflow/operations/create_stream.py b/temporalio/streams/system/workflow/operations/create_stream.py index a1b5e8c6d2..44fba4acd9 100644 --- a/temporalio/streams/system/workflow/operations/create_stream.py +++ b/temporalio/streams/system/workflow/operations/create_stream.py @@ -13,7 +13,6 @@ from ...models import ( CreateStreamRequest, CreateStreamResult, - Execution, StreamIdConflictPolicy, StreamRef, ) @@ -49,7 +48,6 @@ async def _create_stream( async def create_stream( *, id: str, - owner: Execution | None = None, backend: str | None = None, retention: datetime.timedelta | None = None, id_conflict_policy: StreamIdConflictPolicy | None = None, @@ -62,7 +60,6 @@ async def create_stream( async def create_stream( *, id: str, - owner: Execution | None = None, backend: str | None = None, retention: datetime.timedelta | None = None, id_conflict_policy: StreamIdConflictPolicy | None = None, @@ -74,7 +71,6 @@ async def create_stream( async def create_stream( *, id: str, - owner: Execution | None = None, backend: str | None = None, retention: datetime.timedelta | None = None, id_conflict_policy: StreamIdConflictPolicy | None = None, @@ -83,8 +79,6 @@ async def create_stream( """ Args: id: The Stream id, 1 to 4096 bytes. - owner: The run chain or Standalone Activity of a bound Stream. Empty if - standalone. backend: Registered backend name, empty for the default. The Store never reads it. retention: The age after which a Record is dropped. Empty is the Store default. @@ -95,13 +89,11 @@ async def create_stream( body_type = typing.Any stream_ref = StreamRef( id=id, - owner=owner, backend=backend, ) request = CreateStreamRequest( stream_ref=stream_ref, retention=retention, - withdraw_run=_support.withdraw_run(), id_conflict_policy=id_conflict_policy, request_id=_support.request_id(), ) diff --git a/temporalio/streams/writer_system/__init__.py b/temporalio/streams/writer_system/__init__.py index 9a756be679..a5151aaad7 100644 --- a/temporalio/streams/writer_system/__init__.py +++ b/temporalio/streams/writer_system/__init__.py @@ -4,30 +4,16 @@ from __future__ import annotations from .models import ( - ActivityEpoch, AppendRecord, - ClientEpoch, CloseWriterRequest, CloseWriterResponse, CloseWritersRequest, CloseWritersResponse, - Execution, - ExecutionType, FreeWriterRequest, FreeWriterResponse, - LocalActivityEpoch, OpenWriterRequest, Producer, - ProducerEpoch, - ProducerEpochKind, - ProducerEpochKindActivity, - ProducerEpochKindClient, - ProducerEpochKindLocalActivity, - ProducerEpochKindStandaloneActivity, - ProducerEpochKindWorkflow, StreamRef, - Timestamp, - WorkflowTaskEpoch, WriteRecordsRequest, WriteRecordsResponse, WriterResult, @@ -37,32 +23,18 @@ from .services import StreamWriter __all__ = [ - "ActivityEpoch", "AppendRecord", - "ClientEpoch", "CloseWriterRequest", "CloseWriterResponse", "CloseWritersRequest", "CloseWritersResponse", - "Execution", - "ExecutionType", "FreeWriterRequest", "FreeWriterResponse", - "LocalActivityEpoch", "OpenWriterRequest", "Producer", - "ProducerEpoch", - "ProducerEpochKind", - "ProducerEpochKindActivity", - "ProducerEpochKindClient", - "ProducerEpochKindLocalActivity", - "ProducerEpochKindStandaloneActivity", - "ProducerEpochKindWorkflow", "StreamRef", - "Timestamp", - "WorkflowTaskEpoch", + "StreamWriter", "WriteRecordsRequest", "WriteRecordsResponse", "WriterResult", - "StreamWriter", ] diff --git a/temporalio/streams/writer_system/models.py b/temporalio/streams/writer_system/models.py index 47981c2bdb..2e17af546d 100644 --- a/temporalio/streams/writer_system/models.py +++ b/temporalio/streams/writer_system/models.py @@ -1,37 +1,46 @@ -# Written by hand in the shape of nexgen's client output, since the writer has no WIT. Keep it -# close to that output, so a later switch to generated code changes no caller. +# Written by hand in the shape of nexgen's client output, since the writer has no WIT. The shared +# types (StreamRef, StreamWriter, AppendRecord and the Producer models) come from the generated +# system models, so the writer always matches the generated client. Only the writer-specific +# request and response types live here. from __future__ import annotations import dataclasses import datetime -import enum -import typing -import google.protobuf.timestamp_pb2 import typing_extensions -import temporalio.api.common.v1.message_pb2 -import temporalio.api.enums.v1.common_pb2 -import temporalio.api.stream.v1.message_pb2 import temporalio.bridge.proto.streams.streams_pb2 import temporalio.converter import temporalio.nexus.system._support as _support +from temporalio.streams.system.models import ( + AppendRecord, + Producer, + StreamRef, + StreamWriter, + _AppendRecordTransferTypeConverter, + _ProducerTransferTypeConverter, + _StreamRefTransferTypeConverter, + _StreamWriterTransferTypeConverter, +) - -class ExecutionType(enum.IntEnum): - ExecutionTypeUnspecified = 0 - ExecutionTypeWorkflow = 1 - ExecutionTypeActivity = 2 - ExecutionTypeNexusOperation = 3 - - -def _execution_type_from_proto(value: int) -> ExecutionType: - try: - return ExecutionType(value) - except ValueError: - # A newer peer can send a case that this enum doesn't know. - return ExecutionType.ExecutionTypeUnspecified +__all__ = [ + "AppendRecord", + "CloseWriterRequest", + "CloseWriterResponse", + "CloseWritersRequest", + "CloseWritersResponse", + "FreeWriterRequest", + "FreeWriterResponse", + "OpenWriterRequest", + "Producer", + "StreamRef", + "StreamWriter", + "WriteRecordsRequest", + "WriteRecordsResponse", + "WriterResult", + "_ProducerTransferTypeConverter", +] @dataclasses.dataclass(slots=True) @@ -103,603 +112,6 @@ def to_transfer_type( ) # pyright: ignore[reportUnusedCallResult] -@dataclasses.dataclass(slots=True) -class StreamRef: - """Identifies one Stream. The id is opaque to callers.""" - - id: str - owner: Execution | None = None - backend: str | None = None - - -class _StreamRefTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - StreamRef, temporalio.api.stream.v1.message_pb2.StreamRef - ] -): - transfer_type: type[temporalio.api.stream.v1.message_pb2.StreamRef] | None = ( - temporalio.api.stream.v1.message_pb2.StreamRef - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.StreamRef, - type_hint: type[StreamRef], - ) -> StreamRef: - if not value.id: - raise ValueError("missing required field StreamRef.id") - id = value.id - return StreamRef( - id=id, - owner=_ExecutionTransferTypeConverter().from_transfer_type( - value.owner, Execution - ) - if value.HasField("owner") - else None, - backend=value.backend if bool(value.backend) else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: StreamRef, - ) -> temporalio.api.stream.v1.message_pb2.StreamRef: - message = temporalio.api.stream.v1.message_pb2.StreamRef() - message.id = value.id - if value.owner is not None: - message.owner.CopyFrom( - _ExecutionTransferTypeConverter().to_transfer_type(value.owner) - ) - if value.backend is not None: - message.backend = value.backend - return message - - -temporalio.converter.transfer_type_convertible(_StreamRefTransferTypeConverter)( - StreamRef -) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class Execution: - """A Workflow run chain or a Standalone Activity run.""" - - type: ExecutionType | None = None - business_id: str | None = None - run_id: str | None = None - - -class _ExecutionTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - Execution, temporalio.api.common.v1.message_pb2.Execution - ] -): - transfer_type: type[temporalio.api.common.v1.message_pb2.Execution] | None = ( - temporalio.api.common.v1.message_pb2.Execution - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.common.v1.message_pb2.Execution, - type_hint: type[Execution], - ) -> Execution: - return Execution( - type=_execution_type_from_proto(value.type) if value.type != 0 else None, - business_id=value.business_id if bool(value.business_id) else None, - run_id=value.run_id if bool(value.run_id) else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: Execution, - ) -> temporalio.api.common.v1.message_pb2.Execution: - message = temporalio.api.common.v1.message_pb2.Execution() - if value.type is not None: - message.type = temporalio.api.enums.v1.common_pb2.ExecutionType.ValueType( - value.type - ) - if value.business_id is not None: - message.business_id = value.business_id - if value.run_id is not None: - message.run_id = value.run_id - return message - - -temporalio.converter.transfer_type_convertible(_ExecutionTransferTypeConverter)( - Execution -) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True, kw_only=True) -class StreamWriter: - """The Producer of a write, and which earlier Records the write withdraws.""" - - producer: Producer | None = None - previous_completed_task: WorkflowTaskEpoch | None = None - failed_tasks: list[int] - withdraw_run: str | None = None - - -class _StreamWriterTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - StreamWriter, temporalio.api.stream.v1.message_pb2.StreamWriter - ] -): - transfer_type: type[temporalio.api.stream.v1.message_pb2.StreamWriter] | None = ( - temporalio.api.stream.v1.message_pb2.StreamWriter - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.StreamWriter, - type_hint: type[StreamWriter], - ) -> StreamWriter: - failed_tasks = list(value.failed_tasks) - return StreamWriter( - producer=_ProducerTransferTypeConverter().from_transfer_type( - value.producer, Producer - ) - if value.HasField("producer") - else None, - previous_completed_task=_WorkflowTaskEpochTransferTypeConverter().from_transfer_type( - value.previous_completed_task, WorkflowTaskEpoch - ) - if value.HasField("previous_completed_task") - else None, - failed_tasks=failed_tasks, - withdraw_run=value.withdraw_run if bool(value.withdraw_run) else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: StreamWriter, - ) -> temporalio.api.stream.v1.message_pb2.StreamWriter: - message = temporalio.api.stream.v1.message_pb2.StreamWriter() - if value.producer is not None: - message.producer.CopyFrom( - _ProducerTransferTypeConverter().to_transfer_type(value.producer) - ) - if value.previous_completed_task is not None: - message.previous_completed_task.CopyFrom( - _WorkflowTaskEpochTransferTypeConverter().to_transfer_type( - value.previous_completed_task - ) - ) - message.failed_tasks.extend(value.failed_tasks) - if value.withdraw_run is not None: - message.withdraw_run = value.withdraw_run - return message - - -temporalio.converter.transfer_type_convertible(_StreamWriterTransferTypeConverter)( - StreamWriter -) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class Producer: - """The Producer of a batch.""" - - id: str - epoch: ProducerEpoch | None = None - run_id: str | None = None - - -class _ProducerTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - Producer, temporalio.api.stream.v1.message_pb2.Producer - ] -): - transfer_type: type[temporalio.api.stream.v1.message_pb2.Producer] | None = ( - temporalio.api.stream.v1.message_pb2.Producer - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.Producer, - type_hint: type[Producer], - ) -> Producer: - if not value.id: - raise ValueError("missing required field Producer.id") - id = value.id - return Producer( - id=id, - epoch=_ProducerEpochTransferTypeConverter().from_transfer_type( - value.epoch, ProducerEpoch - ) - if value.HasField("epoch") - else None, - run_id=value.run_id if bool(value.run_id) else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: Producer, - ) -> temporalio.api.stream.v1.message_pb2.Producer: - message = temporalio.api.stream.v1.message_pb2.Producer() - message.id = value.id - if value.epoch is not None: - message.epoch.CopyFrom( - _ProducerEpochTransferTypeConverter().to_transfer_type(value.epoch) - ) - if value.run_id is not None: - message.run_id = value.run_id - return message - - -temporalio.converter.transfer_type_convertible(_ProducerTransferTypeConverter)(Producer) - - -@dataclasses.dataclass(slots=True) -class ProducerEpoch: - """The ordered attempt of a Producer. A higher epoch supersedes a lower one.""" - - kind: ProducerEpochKind | None = None - - -class _ProducerEpochTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - ProducerEpoch, temporalio.api.stream.v1.message_pb2.ProducerEpoch - ] -): - transfer_type: type[temporalio.api.stream.v1.message_pb2.ProducerEpoch] | None = ( - temporalio.api.stream.v1.message_pb2.ProducerEpoch - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.ProducerEpoch, - type_hint: type[ProducerEpoch], - ) -> ProducerEpoch: - _oneof_kind: ( - ProducerEpochKindWorkflow - | ProducerEpochKindLocalActivity - | ProducerEpochKindActivity - | ProducerEpochKindStandaloneActivity - | ProducerEpochKindClient - | None - ) - match value.WhichOneof("kind"): - case "workflow": - _oneof_kind = ProducerEpochKindWorkflow( - _WorkflowTaskEpochTransferTypeConverter().from_transfer_type( - value.workflow, WorkflowTaskEpoch - ) - ) - case "local_activity": - _oneof_kind = ProducerEpochKindLocalActivity( - _LocalActivityEpochTransferTypeConverter().from_transfer_type( - value.local_activity, LocalActivityEpoch - ) - ) - case "activity": - _oneof_kind = ProducerEpochKindActivity( - _ActivityEpochTransferTypeConverter().from_transfer_type( - value.activity, ActivityEpoch - ) - ) - case "standalone_activity": - _oneof_kind = ProducerEpochKindStandaloneActivity( - _ActivityEpochTransferTypeConverter().from_transfer_type( - value.standalone_activity, ActivityEpoch - ) - ) - case "client": - _oneof_kind = ProducerEpochKindClient( - _ClientEpochTransferTypeConverter().from_transfer_type( - value.client, ClientEpoch - ) - ) - case None: - _oneof_kind = None - return ProducerEpoch( - kind=_oneof_kind, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: ProducerEpoch, - ) -> temporalio.api.stream.v1.message_pb2.ProducerEpoch: - message = temporalio.api.stream.v1.message_pb2.ProducerEpoch() - if value.kind is not None: - match value.kind: - case ProducerEpochKindWorkflow(): - message.workflow.CopyFrom( - _WorkflowTaskEpochTransferTypeConverter().to_transfer_type( - value.kind.value - ) - ) - case ProducerEpochKindLocalActivity(): - message.local_activity.CopyFrom( - _LocalActivityEpochTransferTypeConverter().to_transfer_type( - value.kind.value - ) - ) - case ProducerEpochKindActivity(): - message.activity.CopyFrom( - _ActivityEpochTransferTypeConverter().to_transfer_type( - value.kind.value - ) - ) - case ProducerEpochKindStandaloneActivity(): - message.standalone_activity.CopyFrom( - _ActivityEpochTransferTypeConverter().to_transfer_type( - value.kind.value - ) - ) - case ProducerEpochKindClient(): - message.client.CopyFrom( - _ClientEpochTransferTypeConverter().to_transfer_type( - value.kind.value - ) - ) - return message - - -temporalio.converter.transfer_type_convertible(_ProducerEpochTransferTypeConverter)( - ProducerEpoch -) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class WorkflowTaskEpoch: - """A Workflow Task attempt of the run.""" - - started_event_id: int | None = None - attempt: int | None = None - started_time: Timestamp | None = None - - -class _WorkflowTaskEpochTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - WorkflowTaskEpoch, temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch - ] -): - transfer_type: ( - type[temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch] | None - ) = temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch, - type_hint: type[WorkflowTaskEpoch], - ) -> WorkflowTaskEpoch: - return WorkflowTaskEpoch( - started_event_id=value.started_event_id - if value.started_event_id != 0 - else None, - attempt=value.attempt if value.attempt != 0 else None, - started_time=_TimestampTransferTypeConverter().from_transfer_type( - value.started_time, Timestamp - ) - if value.HasField("started_time") - else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: WorkflowTaskEpoch, - ) -> temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch: - message = temporalio.api.stream.v1.message_pb2.WorkflowTaskEpoch() - if value.started_event_id is not None: - message.started_event_id = value.started_event_id - if value.attempt is not None: - message.attempt = value.attempt - if value.started_time is not None: - message.started_time.CopyFrom( - _TimestampTransferTypeConverter().to_transfer_type(value.started_time) - ) - return message - - -temporalio.converter.transfer_type_convertible(_WorkflowTaskEpochTransferTypeConverter)( - WorkflowTaskEpoch -) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class Timestamp: - seconds: int | None = None - nanos: int | None = None - - -class _TimestampTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - Timestamp, google.protobuf.timestamp_pb2.Timestamp - ] -): - transfer_type: type[google.protobuf.timestamp_pb2.Timestamp] | None = ( - google.protobuf.timestamp_pb2.Timestamp - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: google.protobuf.timestamp_pb2.Timestamp, - type_hint: type[Timestamp], - ) -> Timestamp: - return Timestamp( - seconds=value.seconds if value.seconds != 0 else None, - nanos=value.nanos if value.nanos != 0 else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: Timestamp, - ) -> google.protobuf.timestamp_pb2.Timestamp: - message = google.protobuf.timestamp_pb2.Timestamp() - if value.seconds is not None: - message.seconds = value.seconds - if value.nanos is not None: - message.nanos = value.nanos - return message - - -temporalio.converter.transfer_type_convertible(_TimestampTransferTypeConverter)( - Timestamp -) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class LocalActivityEpoch: - """A Local Activity attempt. Its Workflow Task orders it first.""" - - task: WorkflowTaskEpoch | None = None - attempt: int | None = None - - -class _LocalActivityEpochTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - LocalActivityEpoch, temporalio.api.stream.v1.message_pb2.LocalActivityEpoch - ] -): - transfer_type: ( - type[temporalio.api.stream.v1.message_pb2.LocalActivityEpoch] | None - ) = temporalio.api.stream.v1.message_pb2.LocalActivityEpoch - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.LocalActivityEpoch, - type_hint: type[LocalActivityEpoch], - ) -> LocalActivityEpoch: - return LocalActivityEpoch( - task=_WorkflowTaskEpochTransferTypeConverter().from_transfer_type( - value.task, WorkflowTaskEpoch - ) - if value.HasField("task") - else None, - attempt=value.attempt if value.attempt != 0 else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: LocalActivityEpoch, - ) -> temporalio.api.stream.v1.message_pb2.LocalActivityEpoch: - message = temporalio.api.stream.v1.message_pb2.LocalActivityEpoch() - if value.task is not None: - message.task.CopyFrom( - _WorkflowTaskEpochTransferTypeConverter().to_transfer_type(value.task) - ) - if value.attempt is not None: - message.attempt = value.attempt - return message - - -temporalio.converter.transfer_type_convertible( - _LocalActivityEpochTransferTypeConverter -)(LocalActivityEpoch) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class ActivityEpoch: - """An Activity or Standalone Activity attempt.""" - - attempt_scheduled_time: Timestamp | None = None - attempt: int | None = None - - -class _ActivityEpochTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - ActivityEpoch, temporalio.api.stream.v1.message_pb2.ActivityEpoch - ] -): - transfer_type: type[temporalio.api.stream.v1.message_pb2.ActivityEpoch] | None = ( - temporalio.api.stream.v1.message_pb2.ActivityEpoch - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.ActivityEpoch, - type_hint: type[ActivityEpoch], - ) -> ActivityEpoch: - return ActivityEpoch( - attempt_scheduled_time=_TimestampTransferTypeConverter().from_transfer_type( - value.attempt_scheduled_time, Timestamp - ) - if value.HasField("attempt_scheduled_time") - else None, - attempt=value.attempt if value.attempt != 0 else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: ActivityEpoch, - ) -> temporalio.api.stream.v1.message_pb2.ActivityEpoch: - message = temporalio.api.stream.v1.message_pb2.ActivityEpoch() - if value.attempt_scheduled_time is not None: - message.attempt_scheduled_time.CopyFrom( - _TimestampTransferTypeConverter().to_transfer_type( - value.attempt_scheduled_time - ) - ) - if value.attempt is not None: - message.attempt = value.attempt - return message - - -temporalio.converter.transfer_type_convertible(_ActivityEpochTransferTypeConverter)( - ActivityEpoch -) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class ClientEpoch: - """A client attempt, which the client picks.""" - - attempt: int | None = None - - -class _ClientEpochTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - ClientEpoch, temporalio.api.stream.v1.message_pb2.ClientEpoch - ] -): - transfer_type: type[temporalio.api.stream.v1.message_pb2.ClientEpoch] | None = ( - temporalio.api.stream.v1.message_pb2.ClientEpoch - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.ClientEpoch, - type_hint: type[ClientEpoch], - ) -> ClientEpoch: - return ClientEpoch( - attempt=value.attempt if value.attempt != 0 else None, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: ClientEpoch, - ) -> temporalio.api.stream.v1.message_pb2.ClientEpoch: - message = temporalio.api.stream.v1.message_pb2.ClientEpoch() - if value.attempt is not None: - message.attempt = value.attempt - return message - - -temporalio.converter.transfer_type_convertible(_ClientEpochTransferTypeConverter)( - ClientEpoch -) # pyright: ignore[reportUnusedCallResult] - - @dataclasses.dataclass(slots=True) class WriterResult: writer_id: int | None = None @@ -794,50 +206,6 @@ def to_transfer_type( )(WriteRecordsRequest) # pyright: ignore[reportUnusedCallResult] -@dataclasses.dataclass(slots=True) -class AppendRecord: - """One Record to append.""" - - body: typing.Any - - -class _AppendRecordTransferTypeConverter( - temporalio.converter.TransferTypeConverter[ - AppendRecord, temporalio.api.stream.v1.message_pb2.AppendRecord - ] -): - transfer_type: type[temporalio.api.stream.v1.message_pb2.AppendRecord] | None = ( - temporalio.api.stream.v1.message_pb2.AppendRecord - ) - - @typing_extensions.override - def from_transfer_type( - self, - value: temporalio.api.stream.v1.message_pb2.AppendRecord, - type_hint: type[AppendRecord], - ) -> AppendRecord: - if not value.HasField("body"): - raise ValueError("missing required field AppendRecord.body") - body = _support.payload_from_proto(value.body) - return AppendRecord( - body=body, - ) - - @typing_extensions.override - def to_transfer_type( - self, - value: AppendRecord, - ) -> temporalio.api.stream.v1.message_pb2.AppendRecord: - message = temporalio.api.stream.v1.message_pb2.AppendRecord() - message.body.CopyFrom(_support.payload_to_proto(value.body)) - return message - - -temporalio.converter.transfer_type_convertible(_AppendRecordTransferTypeConverter)( - AppendRecord -) # pyright: ignore[reportUnusedCallResult] - - @dataclasses.dataclass(slots=True) class WriteRecordsResponse: first_offset: int | None = None @@ -1119,38 +487,3 @@ def to_transfer_type( temporalio.converter.transfer_type_convertible( _CloseWritersResponseTransferTypeConverter )(CloseWritersResponse) # pyright: ignore[reportUnusedCallResult] - - -@dataclasses.dataclass(slots=True) -class ProducerEpochKindWorkflow: - value: WorkflowTaskEpoch - - -@dataclasses.dataclass(slots=True) -class ProducerEpochKindLocalActivity: - value: LocalActivityEpoch - - -@dataclasses.dataclass(slots=True) -class ProducerEpochKindActivity: - value: ActivityEpoch - - -@dataclasses.dataclass(slots=True) -class ProducerEpochKindStandaloneActivity: - value: ActivityEpoch - - -@dataclasses.dataclass(slots=True) -class ProducerEpochKindClient: - value: ClientEpoch - - -ProducerEpochKind = ( - ProducerEpochKindWorkflow - | ProducerEpochKindLocalActivity - | ProducerEpochKindActivity - | ProducerEpochKindStandaloneActivity - | ProducerEpochKindClient -) -"""The epoch of each kind of Producer.""" diff --git a/temporalio/workflow/_streams.py b/temporalio/workflow/_streams.py index 03d126bce9..7a2c7b493f 100644 --- a/temporalio/workflow/_streams.py +++ b/temporalio/workflow/_streams.py @@ -128,16 +128,7 @@ def close_nowait(self) -> None: _workflow.close_nowait(self._ref) async def __aenter__(self) -> Self: - """Use the Stream in ``async with``, which closes it at the end. - - Raises: - StreamError: This run chain can't close the Stream. See :py:meth:`close`. - """ - from temporalio.streams import _workflow - - # Fail before the block, since the close at its end would raise and hide the block's - # own error. - _workflow.check_owner(self._ref) + """Use the Stream in ``async with``, which closes it at the end.""" return self async def __aexit__( diff --git a/tests/contrib/streams/examples/agent_publishes_turns.py b/tests/contrib/streams/examples/agent_publishes_turns.py index c55f0a0186..ca716a3c21 100644 --- a/tests/contrib/streams/examples/agent_publishes_turns.py +++ b/tests/contrib/streams/examples/agent_publishes_turns.py @@ -133,7 +133,5 @@ async def main(client: Client, task_queue: str) -> list[streams.Record[Turn]]: ref = await handle.query(Agent.stream) assert ref is not None # A deserialized ref decodes as JSON, so the id form takes the payload type. - stream = client.get_stream( - ref.id, payload_type=Turn, owner=ref.owner, backend=ref.backend - ) + stream = client.get_stream(ref.id, payload_type=Turn, backend=ref.backend) return [r async for r in stream.read() if isinstance(r, streams.Record)] diff --git a/tests/contrib/streams/examples/chat_app/server.py b/tests/contrib/streams/examples/chat_app/server.py index e2de2d3717..0fc0d5a63f 100644 --- a/tests/contrib/streams/examples/chat_app/server.py +++ b/tests/contrib/streams/examples/chat_app/server.py @@ -38,6 +38,7 @@ from temporalio.converter import DataConverter from temporalio.exceptions import WorkflowAlreadyStartedError from temporalio.service import RPCError, RPCStatusCode +from temporalio.streams.superseded import Superseded, with_superseded from temporalio.worker import Worker from .activities import ModelActivities @@ -255,27 +256,27 @@ async def end(session: SessionDep) -> dict[str, Any]: @app.post("/api/sessions/{session_id}/share") async def share(session_id: str, refs: StreamsDep, chat: ChatDep) -> dict[str, Any]: turns = chat.client.get_stream( - refs.turns.id, - payload_type=Turn, - owner=refs.turns.owner, - backend=refs.turns.backend, + refs.turns.id, payload_type=Turn, backend=refs.turns.backend ) # The snapshot holds the chat up to this offset, which names it. upto = await turns.get_offset() share_id = f"{session_id}-snapshot-{upto}" shown: dict[str, SharedTurn] = {} - async with contextlib.aclosing(turns.read(wait_new_records=False)) as read: + # with_superseded crosses out a failed attempt's Records, so the snapshot copies only what a + # live reader would show. + async with contextlib.aclosing( + with_superseded(turns.read(wait_new_records=False)) + ) as read: async for item in read: if item.offset >= upto: break - match item.kind: - case streams.ItemKind.SUPERSEDED: - for offset in item.withdrawn_offsets: - del shown[str(offset)] - case streams.ItemKind.RECORD: - shown[str(item.offset)] = SharedTurn( - item.data, str(item.offset), source_producer(item.producer) - ) + if isinstance(item, Superseded): + for offset in item.withdrawn_offsets: + del shown[str(offset)] + elif isinstance(item, streams.Record): + shown[str(item.offset)] = SharedTurn( + item.data, str(item.offset), source_producer(item.producer) + ) try: # A strict create, since a share id names one snapshot. async with await chat.client.create_stream( @@ -353,19 +354,13 @@ async def session_events( # init's create can still be in flight when a new chat's tab starts to read. wait = timedelta(seconds=30) turns_stream = chat.client.get_stream( - refs.turns.id, - payload_type=Turn, - owner=refs.turns.owner, - backend=refs.turns.backend, + refs.turns.id, payload_type=Turn, backend=refs.turns.backend ) turns_read = turns_stream.read( offset=streams.StreamOffset.parse(turns) if turns else None, create_wait=wait ) tokens_stream = chat.client.get_stream( - refs.tokens.id, - payload_type=Token, - owner=refs.tokens.owner, - backend=refs.tokens.backend, + refs.tokens.id, payload_type=Token, backend=refs.tokens.backend ) # A tab that joins reads only new tokens, since the turns Stream has the earlier replies. tokens_read = tokens_stream.read( @@ -375,7 +370,7 @@ async def session_events( merged = merge( events("lobby", lobby_read(chat)), events("turns", turns_read), - events("tokens", tokens_read, kept_producers(kept)), + events("tokens", tokens_read, kept_producers(kept), supersede=True), ) async for event in merged: yield event @@ -383,10 +378,7 @@ async def session_events( def lobby_read(chat: Chat) -> streams.Reader[SessionInfo]: lobby = chat.client.get_stream( - chat.lobby.id, - payload_type=SessionInfo, - owner=chat.lobby.owner, - backend=chat.lobby.backend, + chat.lobby.id, payload_type=SessionInfo, backend=chat.lobby.backend ) # The newest 20 chats, then each new one. return lobby.read(last=20) @@ -416,12 +408,22 @@ async def events( name: str, read: streams.Reader[Any], kept: Sequence[streams.Producer] = (), + *, + supersede: bool = False, ) -> AsyncIterator[ServerSentEvent]: + # The tokens read opts into with_superseded, which crosses out a failed attempt's tokens from + # the Producer epochs. The base read's next_offset is where a later read resumes. + items = with_superseded(read) if supersede else read try: - async for item in read: + async for item in items: + # The close ends the read, which the "end" event marks. + if isinstance(item, streams.Closed): + continue data = {"stream": name, **item_json(item, kept)} # Where a later read of this Stream resumes, which the tab saves. - data["next_offset"] = str(read.next_offset) if read.next_offset else None + data["next_offset"] = ( + str(read.next_offset) if read.next_offset is not None else None + ) yield ServerSentEvent(event="item", data=data) yield ServerSentEvent(event="end", data={"stream": name}) except Exception as err: @@ -430,27 +432,24 @@ async def events( def item_json( - item: streams.Record[Any] | streams.Superseded, kept: Sequence[streams.Producer] + item: streams.Record[Any] | Superseded, kept: Sequence[streams.Producer] ) -> dict[str, Any]: # Offsets are past the range of a JavaScript number, so they go as strings. - match item.kind: - case streams.ItemKind.SUPERSEDED: - writer = item.facts.writer - return { - "kind": "superseded", - "offset": str(item.offset), - "producer": producer_json(writer.producer if writer else None), - "withdrawn_offsets": [str(o) for o in item.withdrawn_offsets], - # A tab sends the Producers of the Records that it kept from before a reload. - "withdrawn_kept": [i for i, p in enumerate(kept) if item.withdraws(p)], - } - case streams.ItemKind.RECORD: - return { - "kind": "record", - "offset": str(item.offset), - "producer": producer_json(item.producer), - "data": data_json(item.data), - } + if isinstance(item, Superseded): + return { + "kind": "superseded", + "offset": str(item.offset), + "producer": producer_json(item.producer), + "withdrawn_offsets": [str(o) for o in item.withdrawn_offsets], + # A tab sends the Producers of the Records that it kept from before a reload. + "withdrawn_kept": [i for i, p in enumerate(kept) if item.withdraws(p)], + } + return { + "kind": "record", + "offset": str(item.offset), + "producer": producer_json(item.producer), + "data": data_json(item.data), + } def plain(value: Any) -> Any: @@ -470,7 +469,7 @@ def data_json(value: Any) -> dict[str, Any]: def producer_json(producer: streams.Producer | None) -> dict[str, Any] | None: if producer is None: return None - kind, attempt = epoch_of(producer.epoch) + kind, attempt = epoch_of(producer) return { "id": producer.id, "kind": kind, @@ -483,23 +482,23 @@ def producer_json(producer: streams.Producer | None) -> dict[str, Any] | None: def source_producer(producer: streams.Producer | None) -> SourceProducer | None: if producer is None: return None - kind, attempt = epoch_of(producer.epoch) + kind, attempt = epoch_of(producer) return SourceProducer(producer.id, kind, attempt) -def epoch_of(epoch: streams.ProducerEpoch | None) -> tuple[str, int | None]: - match epoch.kind if epoch else None: - case streams.ProducerEpochKindWorkflow(): +def epoch_of(producer: streams.Producer) -> tuple[str, int | None]: + match producer.kind: + case streams.ProducerKindWorkflow(): return "workflow", None - case streams.ProducerEpochKindLocalActivity(value=local): - return "local activity", local.attempt + case streams.ProducerKindLocalActivity(value=local): + return "local activity", local.epoch.attempt if local.epoch else None case ( - streams.ProducerEpochKindActivity(value=attempt) - | streams.ProducerEpochKindStandaloneActivity(value=attempt) + streams.ProducerKindActivity(value=act) + | streams.ProducerKindStandaloneActivity(value=act) ): - return "activity", attempt.attempt - case streams.ProducerEpochKindClient(value=client): - return "client", client.attempt + return "activity", act.epoch.attempt if act.epoch else None + case streams.ProducerKindClient(value=client): + return "client", client.epoch.attempt if client.epoch else None case _: return "unknown", None diff --git a/tests/contrib/streams/examples/chat_app/test_chat_app.py b/tests/contrib/streams/examples/chat_app/test_chat_app.py index 5dc327784b..edf1bae761 100644 --- a/tests/contrib/streams/examples/chat_app/test_chat_app.py +++ b/tests/contrib/streams/examples/chat_app/test_chat_app.py @@ -1,12 +1,12 @@ """Runs the chat session on the scripted model, on the memory Store and on Redis.""" -import json import uuid import httpx from temporalio import streams from temporalio.client import Client, WorkflowHandle +from temporalio.streams.superseded import Superseded, with_superseded from temporalio.worker import Worker from tests.contrib.streams.examples.chat_app import server from tests.contrib.streams.examples.chat_app.activities import ( @@ -73,7 +73,6 @@ async def answer( read = client.get_stream( refs.turns.id, payload_type=Turn, - owner=refs.turns.owner, backend=refs.turns.backend, ).read() records = await read_until_answer(read, 1) @@ -92,14 +91,12 @@ async def test_turn_and_its_tokens(stream_client: Client): turns = stream_client.get_stream( refs.turns.id, payload_type=Turn, - owner=refs.turns.owner, backend=refs.turns.backend, ) records = [r async for r in turns.read() if isinstance(r, streams.Record)] tokens = stream_client.get_stream( refs.tokens.id, payload_type=Token, - owner=refs.tokens.owner, backend=refs.tokens.backend, ) deltas = [r.data async for r in tokens.read() if isinstance(r, streams.Record)] @@ -173,7 +170,6 @@ async def test_message_during_a_run_joins_the_next_round(stream_client: Client): read = stream_client.get_stream( refs.turns.id, payload_type=Turn, - owner=refs.turns.owner, backend=refs.turns.backend, ).read() records = [] @@ -219,7 +215,6 @@ async def test_sub_agent_writes_its_own_streams(stream_client: Client): child = stream_client.get_stream( link.agent.turns.id, payload_type=Turn, - owner=link.agent.turns.owner, backend=link.agent.turns.backend, ) child_turns = [r.data async for r in child.read() if isinstance(r, streams.Record)] @@ -234,7 +229,6 @@ async def test_sub_agent_writes_its_own_streams(stream_client: Client): tokens = stream_client.get_stream( link.agent.tokens.id, payload_type=Token, - owner=link.agent.tokens.owner, backend=link.agent.tokens.backend, ) assert [r async for r in tokens.read() if isinstance(r, streams.Record)] @@ -243,7 +237,7 @@ async def test_sub_agent_writes_its_own_streams(stream_client: Client): async def failed_attempt( client: Client, -) -> tuple[SessionStreams, list[streams.Record[Token] | streams.Superseded]]: +) -> tuple[SessionStreams, list[streams.Record[Token] | Superseded | streams.Closed]]: """The refs of a session whose one round failed its first attempt, and the token items.""" async with chat_worker(client) as worker: handle = await start(client, worker) @@ -256,15 +250,14 @@ async def failed_attempt( tokens = client.get_stream( refs.tokens.id, payload_type=Token, - owner=refs.tokens.owner, backend=refs.tokens.backend, ) - return refs, [item async for item in tokens.read()] + return refs, [item async for item in with_superseded(tokens.read())] async def test_retry_withdraws_the_failed_attempt(stream_client: Client): _, items = await failed_attempt(stream_client) - [superseded] = [i for i in items if isinstance(i, streams.Superseded)] + [superseded] = [i for i in items if isinstance(i, Superseded)] records = [i for i in items if isinstance(i, streams.Record)] withdrawn = [r for r in records if r.offset in superseded.withdrawn_offsets] kept = [r for r in records if r.offset not in superseded.withdrawn_offsets] @@ -277,36 +270,6 @@ async def test_retry_withdraws_the_failed_attempt(stream_client: Client): ) -async def test_server_checks_kept_tokens_with_withdraws(stream_client: Client): - refs, items = await failed_attempt(stream_client) - shown = [i for i in items if isinstance(i, streams.Record)][:FAIL_AFTER] - # A tab that reloaded after it showed the failed attempt's tokens sends their Producer, and - # its read resumes after them. - producer = server.producer_json(shown[0].producer) - assert producer is not None - kept = server.kept_producers(json.dumps([producer["wire"]])) - tokens = stream_client.get_stream( - refs.tokens.id, - payload_type=Token, - owner=refs.tokens.owner, - backend=refs.tokens.backend, - ) - # Read the shown tokens again to get where the tab's read resumes, as it would from the - # next_offset that it saved. - resume_read = tokens.read(wait_new_records=False) - for _ in range(len(shown)): - await anext(resume_read) - resume = resume_read.next_offset - await resume_read.aclose() - [superseded] = [ - server.item_json(item, kept) - async for item in tokens.read(offset=resume) - if isinstance(item, streams.Superseded) - ] - assert superseded["withdrawn_offsets"] == [] - assert superseded["withdrawn_kept"] == [0] - - async def test_resume_from_a_saved_offset(stream_client: Client): async with chat_worker(stream_client) as worker: handle = await start(stream_client, worker) @@ -314,7 +277,6 @@ async def test_resume_from_a_saved_offset(stream_client: Client): turns = stream_client.get_stream( refs.turns.id, payload_type=Turn, - owner=refs.turns.owner, backend=refs.turns.backend, ) await handle.execute_update(ChatSession.send, UserMessage("message 1")) @@ -361,7 +323,6 @@ async def test_share_again_is_a_strict_create_of_an_existing_id(stream_client: C turns = stream_client.get_stream( refs.turns.id, payload_type=Turn, - owner=refs.turns.owner, backend=refs.turns.backend, ) read = turns.read() diff --git a/tests/contrib/streams/examples/ui_backend_reads.py b/tests/contrib/streams/examples/ui_backend_reads.py index ecf43bcce1..cd4ff7385c 100644 --- a/tests/contrib/streams/examples/ui_backend_reads.py +++ b/tests/contrib/streams/examples/ui_backend_reads.py @@ -15,6 +15,7 @@ from temporalio import activity, streams, workflow from temporalio.client import Client +from temporalio.streams.superseded import Superseded, with_superseded from temporalio.worker import Worker @@ -83,14 +84,19 @@ async def main(client: Client, task_queue: str) -> Screen: server = asyncio.create_task(chat_server(client, chat_id, task_queue)) screen = Screen() stream = client.get_stream(chat_id, payload_type=Turn) # by id, with no call - async for r in stream.read( - offset=saved_offset(), # wherever the app keeps it, or streams.END - create_wait=timedelta(seconds=10), # the creator may not have created it + # The UI opts into supersession, so a retried attempt's Records drop from the view. + async for item in with_superseded( + stream.read( + offset=saved_offset(), # wherever the app keeps it, or streams.END + create_wait=timedelta( + seconds=10 + ), # the creator may not have created it + ) ): - match r.kind: # the kind narrows the item's type - case streams.ItemKind.SUPERSEDED: - screen.unrender(r.withdrawn_offsets) - case _: - screen.render(r.data, r.offset) # a Turn + if isinstance(item, Superseded): + screen.unrender(item.withdrawn_offsets) + elif isinstance(item, streams.Record): + screen.render(item.data, item.offset) # a Turn + # A Closed item ends the view. await server return screen diff --git a/tests/contrib/streams/test_activity_streams.py b/tests/contrib/streams/test_activity_streams.py index c26277cfe0..8e21878549 100644 --- a/tests/contrib/streams/test_activity_streams.py +++ b/tests/contrib/streams/test_activity_streams.py @@ -29,6 +29,7 @@ from temporalio.contrib.stream.testing import MemoryStreamBackend from temporalio.exceptions import ActivityError, ApplicationError, CancelledError from temporalio.streams import _activity, _activity_stream +from temporalio.streams.superseded import Superseded, with_superseded from temporalio.streams.system import models from tests.contrib.streams._support import stream_status from tests.helpers import new_worker @@ -60,6 +61,14 @@ async def read(handle: WorkflowHandle[Any, Any], name: str = "tokens") -> list[o return [item async for item in stream.read(wait_new_records=False)] +async def read_superseded( + handle: WorkflowHandle[Any, Any], name: str = "tokens" +) -> list[object]: + # The opt-in library adds the Superseded items that an Activity retry withdraws. + stream = handle._client.get_stream(f"{handle.id}-{name}", payload_type=str) + return [item async for item in with_superseded(stream.read(wait_new_records=False))] + + async def closed(handle: WorkflowHandle[Any, Any], name: str = "tokens") -> bool: status = await stream_status(handle._client, f"{handle.id}-{name}") return status == models.StreamStatus.StreamStatusClosed @@ -201,21 +210,6 @@ async def run(self, key: str) -> None: ) -async def test_retry_withdraws_a_received_stream(stream_client: Client): - key = str(uuid.uuid4()) - async with new_worker(stream_client, Retrying, activities=[flaky]) as worker: - handle = await stream_client.start_workflow( - Retrying.run, key, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue - ) - await handle.result() - items = await read(handle) - superseded = [item for item in items if isinstance(item, streams.Superseded)] - # The second attempt writes nothing, and its start still withdraws the first one's Record. - assert len(superseded) == 1 - withdrawn = [r.data for r in records(items) if superseded[0].withdraws(r)] - assert withdrawn == ["attempt-1"] - - @dataclass class FinishedStreams: key: str @@ -322,7 +316,7 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( task_queue=worker.task_queue, ) stream = stream_client.get_stream(f"{handle.id}-tokens", payload_type=str) - live = stream.read(create_wait=timedelta(seconds=20)) + live = with_superseded(stream.read(create_wait=timedelta(seconds=20))) reconnects = stream.read(create_wait=timedelta(seconds=20)) first = await anext(live) before = await anext(reconnects) @@ -332,16 +326,17 @@ async def test_reader_withdraws_a_shown_record_after_a_reconnect( await reconnects.aclose() shown[key].set() live_items = [first, *[item async for item in live]] - resumed = [item async for item in stream.read(offset=resume_at)] + resumed = [ + item async for item in with_superseded(stream.read(offset=resume_at)) + ] assert isinstance(first, streams.Record) and first.data == "attempt-1" assert isinstance(before, streams.Record) and before.offset == first.offset - [live_superseded] = [i for i in live_items if isinstance(i, streams.Superseded)] + [live_superseded] = [i for i in live_items if isinstance(i, Superseded)] assert live_superseded.withdrawn_offsets == (first.offset,) - # A new read got no Record of the failed attempt, so the item lists no offset. The kept - # Record is still withdrawn. - [superseded] = [i for i in resumed if isinstance(i, streams.Superseded)] - assert superseded.withdrawn_offsets == () - assert superseded.withdraws(before) + # A fresh read after the failed attempt has no Record of it, so it adds no Superseded item. + # A reader that kept the shown Record checks it with withdraws(). + assert live_superseded.withdraws(before) + assert not any(isinstance(i, Superseded) for i in resumed) assert [r.data for r in records(resumed)] == ["attempt-2"] @@ -668,14 +663,10 @@ async def test_only_the_workers_client_writes_as_the_attempt(stream_client: Clie ) await handle.result() items = await read(handle) - kinds = { - r.data: type(r.producer.epoch.kind) - for r in records(items) - if r.producer and r.producer.epoch - } + kinds = {r.data: type(r.producer.kind) for r in records(items) if r.producer} assert kinds == { - "worker client": streams.ProducerEpochKindActivity, - "other client": streams.ProducerEpochKindClient, + "worker client": streams.ProducerKindActivity, + "other client": streams.ProducerKindClient, } @@ -777,10 +768,9 @@ async def test_attempt_end_flushes_and_retry_finds_nested_streams( assert await handle.result() == ["a", "b", "attempt-1"] items = await read(handle) other = await read(handle, "other") - for read_items in (items, other): - superseded = [i for i in read_items if isinstance(i, streams.Superseded)] - withdrawn = [r.data for r in records(read_items) if superseded[0].withdraws(r)] - assert withdrawn == ["attempt-1"] + # The retry writes nothing, so the failed attempt's Records stay visible on both Streams. + assert "attempt-1" in [r.data for r in records(items)] + assert [r.data for r in records(other)] == ["attempt-1"] @contextlib.contextmanager @@ -992,8 +982,8 @@ async def test_only_a_run_that_uses_streams_keys_its_activities(stream_client: C ) await receives.result() assert await keyed(receives, "plain") == [True] - # The next run gets the Stream as its argument. Its retried Activity withdraws the first - # attempt's Records, as in the run that created the Stream. + # The next run gets the Stream as its argument, so its Activities carry the Producer key + # too, as in the run that created the Stream. continued_key = str(uuid.uuid4()) continued = await stream_client.start_workflow( RetriesAfterContinueAsNew.run, @@ -1003,9 +993,3 @@ async def test_only_a_run_that_uses_streams_keys_its_activities(stream_client: C ) await continued.result() assert await keyed(continued, "flaky") == [True] - items = await read(continued) - superseded = [item for item in items if isinstance(item, streams.Superseded)] - assert len(superseded) == 1 - assert [r.data for r in records(items) if superseded[0].withdraws(r)] == [ - "attempt-1" - ] diff --git a/tests/contrib/streams/test_bindings.py b/tests/contrib/streams/test_bindings.py index 24ef498696..e95c2004ce 100644 --- a/tests/contrib/streams/test_bindings.py +++ b/tests/contrib/streams/test_bindings.py @@ -124,31 +124,6 @@ async def test_a_poll_resumes_after_the_last_item_it_gave(): assert (resumed.from_offset, resumed.last) == (5, None) -async def test_a_poll_resumes_at_a_superseded_item_and_keeps_its_record(): - superseded = models.StreamRecord[str]( - item=models.StreamItemSuperseded( - models.StreamSuperseded( - writer=models.StreamWriter( - producer=models.Producer(id="act:run:a:k"), failed_tasks=[] - ) - ) - ), - offset=7, - ) - nexus_client = FakeNexusClient( - models.ReadRecordsResponse(records=[superseded, _data(7)], next_offset=8), - ) - stream = FakeService(nexus_client).get_stream( - models.StreamRef(id="wf:run"), body_type=str - ) - items = stream.iter_read_items() - assert (await anext(items)).offset == 7 - # The first Record of the batch has the Superseded item's offset and comes next, so a read - # that resumes here must not skip it. - assert items.next_request is not None - assert items.next_request.from_offset == 7 - - async def test_start_now_runs_a_call_to_its_first_wait_at_once(): steps: list[str] = [] gate: asyncio.Future[str] = asyncio.get_running_loop().create_future() diff --git a/tests/contrib/streams/test_facade.py b/tests/contrib/streams/test_facade.py index b12be9a8ec..a6c990a2ed 100644 --- a/tests/contrib/streams/test_facade.py +++ b/tests/contrib/streams/test_facade.py @@ -85,8 +85,8 @@ async def test_client_writes_reads_and_closes(client: Client): turn, token = items assert isinstance(turn, streams.Record) and turn.data == Turn("model") assert isinstance(token, streams.Record) and token.data == Turn("user", "hello") - assert turn.producer is not None and turn.producer.epoch is not None - assert isinstance(turn.producer.epoch.kind, streams.ProducerEpochKindClient) + assert turn.producer is not None + assert isinstance(turn.producer.kind, streams.ProducerKindClient) assert extension.namespaces == {client.namespace} resumed = [item async for item in stream.read(offset=token.offset)] diff --git a/tests/contrib/streams/test_pydantic.py b/tests/contrib/streams/test_pydantic.py index 2169792099..25bd508f38 100644 --- a/tests/contrib/streams/test_pydantic.py +++ b/tests/contrib/streams/test_pydantic.py @@ -58,8 +58,6 @@ async def test_refs_in_models_round_trip_on_the_memory_store(client: Client): ) result = await handle.result() assert result.text == "HI THERE" - # A Workflow creates only standalone Streams, so the ref has no owner. - assert result.stream.owner is None assert result.stream.id == f"{handle.id}-tokens" assert result.stream.backend == "memory" stream = client.get_stream(result.stream) diff --git a/tests/contrib/streams/test_reader.py b/tests/contrib/streams/test_reader.py index 72d310b2f8..6b85833f34 100644 --- a/tests/contrib/streams/test_reader.py +++ b/tests/contrib/streams/test_reader.py @@ -1,4 +1,3 @@ -from collections.abc import AsyncGenerator from datetime import timedelta from typing import Any, cast @@ -7,8 +6,7 @@ import temporalio.common import temporalio.converter -from temporalio.streams import StreamNotFoundError, StreamOffset, _backend, _reader -from temporalio.streams._items import Record, Superseded +from temporalio.streams import StreamNotFoundError, _backend, _reader from temporalio.streams.system import models @@ -49,39 +47,6 @@ async def test_a_waiting_read_carries_the_offset_until_records_come(): assert timedelta() < second.wait <= timedelta(seconds=60) -def _activity(id: str, attempt: int) -> models.Producer: - return models.Producer( - id=id, - epoch=models.ProducerEpoch( - kind=models.ProducerEpochKindActivity(models.ActivityEpoch(attempt=attempt)) - ), - ) - - -async def test_superseded_lists_each_withdrawn_offset_once(): - failed = _activity("act:run:1:1", 1) - other = _activity("act:run:2:2", 1) - # A batch has consecutive offsets, and other Producers' Records come between batches. - written = [(10, failed), (11, failed), (12, failed), (13, other), (20, failed)] - facts = models.StreamSuperseded( - writer=models.StreamWriter(producer=_activity(failed.id, 2), failed_tasks=[]) - ) - - async def items() -> AsyncGenerator[_reader.Item, None]: - for offset, producer in written: - yield Record( - data=None, offset=StreamOffset._from_wire(offset), producer=producer - ) - yield Superseded(offset=StreamOffset._from_wire(30), facts=facts) - yield Superseded(offset=StreamOffset._from_wire(31), facts=facts) - - read: _reader.Reader[None] = _reader.Reader(items(), _reader.Position(0)) - superseded = [item async for item in read if isinstance(item, Superseded)] - assert [ - tuple(str(offset) for offset in item.withdrawn_offsets) for item in superseded - ] == [("10", "11", "12", "20"), ()] - - class _Missing(_backend.StreamServiceCalls): """Answers each read at once with NOT_FOUND, as a Store that ignores create_wait.""" diff --git a/tests/contrib/streams/test_redis.py b/tests/contrib/streams/test_redis.py index 2811e9fd49..16dba767dd 100644 --- a/tests/contrib/streams/test_redis.py +++ b/tests/contrib/streams/test_redis.py @@ -71,7 +71,7 @@ async def run(self, key: str) -> None: await stream.close() -async def test_failed_workflow_task_is_withdrawn(stream_client: Client): +async def test_failed_workflow_task_republishes_its_records(stream_client: Client): key = str(uuid.uuid4()) # The test's counter must keep its value across the task's retry, so the Workflow has no # sandbox. @@ -93,12 +93,9 @@ async def test_failed_workflow_task_is_withdrawn(stream_client: Client): await stream_status(stream_client, stream_id) == models.StreamStatus.StreamStatusClosed ) - # The retried task writes "after" again, and withdraws the copy of the failed task. + # The retried task writes "after" again, and both copies stay visible. written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] assert [record.data for record in written] == ["before", "after", "after"] - withdrawn = [r for r in written if any(i.withdraws(r) for i in superseded)] - assert withdrawn == [written[1]] @dataclass @@ -132,15 +129,16 @@ async def run(self, key: str) -> None: def _task_of(producer: streams.Producer | None) -> tuple[int, int]: - assert producer is not None and producer.epoch is not None - kind = producer.epoch.kind - assert isinstance(kind, streams.ProducerEpochKindLocalActivity) - task = kind.value.task + assert producer is not None + kind = producer.kind + assert isinstance(kind, streams.ProducerKindLocalActivity) + epoch = kind.value.epoch + task = epoch.task if epoch else None assert task is not None return (task.attempt or 0, task.started_event_id or 0) -async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Client): +async def test_rerun_local_activity_keeps_both_attempts(stream_client: Client): async with new_worker( stream_client, LocalThenFailTask, @@ -158,13 +156,13 @@ async def test_rerun_local_activity_withdraws_its_failed_task(stream_client: Cli stream_client.get_stream(f"{handle.id}-tokens", payload_type=str) ) written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] - visible = [r for r in written if not any(i.withdraws(r) for i in superseded)] - assert [r.data for r in visible] == ["2", "3"] + # A Local Activity is never superseded, so both the failed task's and the rerun's Records + # stay visible. + assert [r.data for r in written] == ["0", "1", "2", "3"] # The rerun keeps the Producer id, and its epoch has the retry task, which orders it later. - first, second = written[0].producer, visible[0].producer - assert first is not None and second is not None and first.id == second.id - assert _task_of(first) < _task_of(second) + first, last = written[0].producer, written[-1].producer + assert first is not None and last is not None and first.id == last.id + assert _task_of(first) < _task_of(last) @workflow.defn @@ -186,9 +184,9 @@ async def run(self, key: str) -> None: await stream.close() -async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Client): - # The retry task skips the Local Activity, so only its own write can withdraw "0 1". The - # write also uses the skipped call's key. +async def test_skipped_local_activitys_records_stay_visible(stream_client: Client): + # The retry task skips the Local Activity. A Local Activity is never superseded, so the + # skipped call's "0 1" stay visible alongside the retry's own write. async with new_worker( stream_client, LocalOnlyInFailedTask, @@ -209,9 +207,7 @@ async def test_later_write_withdraws_a_skipped_local_activity(stream_client: Cli == models.StreamStatus.StreamStatusClosed ) written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] - visible = [r for r in written if not any(s.withdraws(r) for s in superseded)] - assert [r.data for r in visible] == ["other"] + assert [r.data for r in written] == ["0", "1", "other"] @dataclass @@ -262,42 +258,6 @@ async def run(self, prompt: str) -> str: return reply -async def test_resume_after_a_superseded_item_keeps_its_record(stream_client: Client): - key = str(uuid.uuid4()) - async with new_worker( - stream_client, - PublishThenFailTask, - workflow_runner=UnsandboxedWorkflowRunner(), - ) as worker: - handle = await stream_client.start_workflow( - PublishThenFailTask.run, - key, - id=f"wf-{uuid.uuid4()}", - task_queue=worker.task_queue, - ) - await handle.result() - stream = stream_of(stream_client, handle) - items = [item async for item in stream.read(wait_new_records=False)] - # The retried task's batch has its Superseded item and its first Record in one entry. - record_offsets = {r.offset for r in records(items)} - [shared] = [ - i - for i in items - if isinstance(i, streams.Superseded) and i.offset in record_offsets - ] - reader = stream.read(wait_new_records=False) - async for item in reader: - if item == shared: - break - resume_at = reader.next_offset - await reader.aclose() - assert resume_at == shared.offset - resumed = [ - item async for item in stream.read(offset=resume_at, wait_new_records=False) - ] - assert [r.data for r in records(resumed) if r.offset == shared.offset] == ["after"] - - @activity.defn async def noop() -> None: pass @@ -636,18 +596,12 @@ async def test_final_failure_under_a_retry_policy( # The end of the chain doesn't close the Stream. assert not await closed(stream_client, handle, "tokens") written = records(items) - superseded = [item for item in items if isinstance(item, streams.Superseded)] if non_retryable: # A non-retryable failure ends the chain on attempt 1. assert [r.data for r in written] == ["run-1"] - assert superseded == [] return - # Each retry run withdraws the Records of the run before it. + # Each retry run's Records stay visible alongside the runs before it. assert [r.data for r in written] == ["run-1", "run-2", "run-3"] - assert len(superseded) == 2 - assert [r.data for r in written if superseded[0].withdraws(r)] == ["run-1"] - assert [r.data for r in written if superseded[1].withdraws(r)] == ["run-2"] - assert not any(s.withdraws(written[-1]) for s in superseded) async def test_reader_stops_on_its_own_timer_and_resumes(stream_client: Client): @@ -813,8 +767,9 @@ async def test_write_to_a_closed_stream_raises_at_the_await(stream_client: Clien ) == "STREAM_ERROR_REASON_CLOSED" ) - # The close wrote no item, so a read of the closed Stream ends at once with none. - assert [item async for item in stream.read()] == [] + # The close wrote no Record, so a read of the closed Stream yields only the Closed item. + items = [item async for item in stream.read()] + assert len(items) == 1 and isinstance(items[0], streams.Closed) @workflow.defn @@ -850,5 +805,3 @@ async def test_retry_of_a_continued_run_writes_the_same_stream(stream_client: Cl assert await closed(stream_client, handle, "tokens") written = records(items) assert [r.data for r in written] == ["first", "attempt-1", "attempt-2"] - superseded = [item for item in items if isinstance(item, streams.Superseded)] - assert [r.data for r in written if superseded[0].withdraws(r)] == ["attempt-1"] diff --git a/tests/contrib/streams/test_refs.py b/tests/contrib/streams/test_refs.py index a35a8c9e71..6a562cb548 100644 --- a/tests/contrib/streams/test_refs.py +++ b/tests/contrib/streams/test_refs.py @@ -7,14 +7,11 @@ import pydantic from google.protobuf import json_format -import temporalio.api.common.v1 -import temporalio.api.enums.v1 import temporalio.api.stream.v1 import temporalio.common from temporalio import streams from temporalio.contrib.pydantic import pydantic_data_converter from temporalio.converter import DataConverter -from temporalio.streams import _items @dataclass @@ -22,11 +19,8 @@ class Turn: kind: str -OWNER = streams.Execution( - type=streams.ExecutionType.ExecutionTypeWorkflow, business_id="wf", run_id="run" -) BOUND: streams.StreamRef[Turn] = streams.StreamRef( - id="session-7-turns", owner=OWNER, backend="redis" + id="session-7-turns", backend="redis" ) @@ -35,31 +29,21 @@ def test_ref_is_the_proto_json_of_the_ref_message(): payload = converter.to_payload(BOUND) message = temporalio.api.stream.v1.StreamRef( id="session-7-turns", - owner=temporalio.api.common.v1.Execution( - type=temporalio.api.enums.v1.ExecutionType.EXECUTION_TYPE_WORKFLOW, - business_id="wf", - run_id="run", - ), backend="redis", ) assert json.loads(payload.data) == json_format.MessageToDict(message) assert payload.metadata["encoding"] == b"json/plain" - assert payload.data == ( - b'{"backend":"redis","id":"session-7-turns",' - b'"owner":{"businessId":"wf","runId":"run","type":"EXECUTION_TYPE_WORKFLOW"}}' - ) + assert payload.data == b'{"backend":"redis","id":"session-7-turns"}' assert json.loads(converter.to_payload(streams.StreamRef(id="s")).data) == { "id": "s" } def test_ref_has_the_fields_of_the_wire_model(): - assert BOUND.id == "session-7-turns" - assert BOUND.owner is not None and BOUND.owner.business_id == "wf" - assert BOUND.owner.run_id == "run" and BOUND.backend == "redis" + assert BOUND.id == "session-7-turns" and BOUND.backend == "redis" assert streams.StreamRef(id="s") == streams.StreamRef("s") assert streams.StreamRef(id="s") != streams.StreamRef(id="s", backend="redis") - same: streams.StreamRef[Turn] = streams.StreamRef("session-7-turns", OWNER, "redis") + same: streams.StreamRef[Turn] = streams.StreamRef("session-7-turns", "redis") assert same == BOUND and len({BOUND, same}) == 1 @@ -85,10 +69,9 @@ class CallModelInput: def test_ref_in_a_dataclass_field(): converter = DataConverter.default.payload_converter payload = converter.to_payload(CallModelInput("hi", BOUND)) - assert json.loads(payload.data)["stream"]["owner"]["businessId"] == "wf" + assert json.loads(payload.data)["stream"]["id"] == "session-7-turns" assert payload.data == ( - b'{"prompt":"hi","stream":{"backend":"redis","id":"session-7-turns",' - b'"owner":{"businessId":"wf","runId":"run","type":"EXECUTION_TYPE_WORKFLOW"}}}' + b'{"prompt":"hi","stream":{"backend":"redis","id":"session-7-turns"}}' ) decoded = converter.from_payload(payload, CallModelInput) assert decoded.stream == BOUND @@ -102,15 +85,7 @@ class CallModelModel(pydantic.BaseModel): def test_ref_in_a_pydantic_field(): model = CallModelModel(prompt="hi", stream=BOUND) dumped = json.loads(model.model_dump_json()) - assert dumped["stream"] == { - "id": "session-7-turns", - "owner": { - "type": "EXECUTION_TYPE_WORKFLOW", - "businessId": "wf", - "runId": "run", - }, - "backend": "redis", - } + assert dumped["stream"] == {"id": "session-7-turns", "backend": "redis"} loaded = CallModelModel.model_validate_json(model.model_dump_json()) assert loaded.stream == BOUND converter = pydantic_data_converter.payload_converter @@ -119,85 +94,30 @@ def test_ref_in_a_pydantic_field(): assert decoded.stream == BOUND -def test_a_ref_keeps_an_owner_of_any_type(): - # Data from a later stage or another SDK can carry a Standalone Activity's run. - owner = streams.Execution( - type=streams.ExecutionType.ExecutionTypeActivity, business_id="a", run_id="r" - ) - ref: streams.StreamRef[str] = streams.StreamRef(id="s", owner=owner) - converter = DataConverter.default.payload_converter - decoded = converter.from_payload(converter.to_payload(ref), streams.StreamRef[str]) - assert decoded == ref and decoded.owner == owner - - -def _producer(id: str, attempt: int) -> streams.Producer: - return streams.Producer( - id=id, - epoch=streams.ProducerEpoch( - kind=streams.ProducerEpochKindActivity( - streams.ActivityEpoch(attempt=attempt) - ) - ), - run_id="run", - ) - - def test_read_items_have_a_kind_that_narrows_their_type(): record = streams.Record( data=Turn("model"), offset=streams.StreamOffset.parse("3"), - producer=_producer("a", 1), + producer=None, ) - superseded = streams.Superseded( + closed = streams.Closed( offset=streams.StreamOffset.parse("4"), - facts=streams.StreamSuperseded( - writer=streams.StreamWriter(producer=_producer("a", 2), failed_tasks=[]) - ), + status=streams.StreamCloseStatus.OWNER_COMPLETED, + failure=None, ) - items: list[streams.Record[Turn] | streams.Superseded] = [record, superseded] + items: list[streams.Record[Turn] | streams.Closed] = [record, closed] shown: list[Turn] = [] - withdrawn: list[bool] = [] + ended: list[streams.StreamCloseStatus] = [] for item in items: - match item.kind: - case streams.ItemKind.SUPERSEDED: - withdrawn.append(item.withdraws(record)) + match item.kind: # the kind narrows the item's type + case streams.ItemKind.CLOSED: + ended.append(item.status) case _: shown.append(item.data) - assert shown == [Turn("model")] and withdrawn == [True] + assert shown == [Turn("model")] + assert ended == [streams.StreamCloseStatus.OWNER_COMPLETED] assert record.kind is streams.ItemKind.RECORD - - -def test_superseded_withdraws_by_the_producer_of_an_earlier_record(): - superseded = streams.Superseded( - offset=streams.StreamOffset.parse("9"), - facts=streams.StreamSuperseded( - writer=streams.StreamWriter(producer=_producer("a", 2), failed_tasks=[]) - ), - ) - # An app that resumes keeps only the Producer of each Record that it showed. - assert superseded.withdraws(_producer("a", 1)) - assert not superseded.withdraws(_producer("a", 2)) - assert not superseded.withdraws(_producer("b", 1)) - - -def test_a_client_producer_is_never_withdrawn(): - client = streams.Producer( - id="client:me", - epoch=streams.ProducerEpoch( - kind=streams.ProducerEpochKindClient(streams.ClientEpoch(attempt=1)) - ), - ) - newer = streams.Producer( - id="client:me", - epoch=streams.ProducerEpoch( - kind=streams.ProducerEpochKindClient(streams.ClientEpoch(attempt=2)) - ), - ) - facts = streams.StreamSuperseded( - writer=streams.StreamWriter(producer=newer, failed_tasks=[]) - ) - assert _items.is_client(client) - assert not _items.withdraws(facts, client) + assert closed.kind is streams.ItemKind.CLOSED def test_temporalio_common_has_no_stream_api(): diff --git a/tests/contrib/streams/test_stream_client.py b/tests/contrib/streams/test_stream_client.py index 5fb0d4ccb3..7d59a3f900 100644 --- a/tests/contrib/streams/test_stream_client.py +++ b/tests/contrib/streams/test_stream_client.py @@ -110,25 +110,6 @@ async def test_next_call_raises_a_write_error_that_no_caller_got(client: Client) assert await stream.get_offset() == _BEGINNING -async def test_a_client_cant_close_a_stream_bound_to_a_workflow(client: Client): - client, extension = fake_client(client) - await client.create_stream("bound", payload_type=str) - owner = streams.Execution( - type=streams.ExecutionType.ExecutionTypeWorkflow, - business_id="wf", - run_id="run", - ) - ref: streams.StreamRef[str] = streams.StreamRef(id="bound", owner=owner) - bound = client.get_stream(ref) - with pytest.raises(streams.StreamError) as not_owner: - await bound.close() - assert not_owner.value.reason == StreamErrorReason.STREAM_ERROR_REASON_NOT_OWNER - with pytest.raises(streams.StreamError): - async with bound: - pass - assert not extension.streams["bound"].closed - - @workflow.defn class HandsOutStream: @workflow.run @@ -386,22 +367,9 @@ async def test_record_over_the_store_limit_raises_too_large(stream_client: Clien assert await stream.get_offset() == _BEGINNING -async def test_read_keeps_no_offsets_of_client_records(stream_client: Client): - # Nothing withdraws a client's Records, so a long UI read of them keeps no state. - stream = await stream_client.create_stream(f"chat-{uuid.uuid4()}", payload_type=str) - await stream.append("a", "b", "c") - read = stream.read(wait_new_records=False) - assert [item.data async for item in read if isinstance(item, streams.Record)] == [ - "a", - "b", - "c", - ] - assert read._shown == {} - - async def test_create_of_an_expired_stream_raises(client: Client): - # Retention is a backend setting now. An open standalone Stream expires one Retention - # after its last write. + # Retention is a backend setting now. safety_ttl keeps an open Stream, so only a closed + # Stream expires, one Retention after its close. config = client.config() config["stream_backends"] = [ MemoryStreamBackend(retention=timedelta(milliseconds=200), default=True) @@ -410,6 +378,7 @@ async def test_create_of_an_expired_stream_raises(client: Client): stream_id = f"expired-{uuid.uuid4()}" stream = await one.create_stream(stream_id, payload_type=str) await stream.append("old") + await stream.close() await asyncio.sleep(0.5) with pytest.raises(streams.StreamError) as expired: await one.create_stream( @@ -569,8 +538,7 @@ async def test_client_reads_a_workflow_stream(stream_client: Client): "hi", "there", ] - # A Workflow creates only standalone Streams, so the ref has no owner. - assert reader.ref.owner is None and reader.ref.backend is not None + assert reader.ref.backend is not None async def test_empty_stream_id_raises_with_no_call(client: Client): diff --git a/tests/contrib/streams/test_superseded.py b/tests/contrib/streams/test_superseded.py new file mode 100644 index 0000000000..1a3c69cdda --- /dev/null +++ b/tests/contrib/streams/test_superseded.py @@ -0,0 +1,144 @@ +"""Unit coverage for the opt-in ``with_superseded`` library. + +The base read gives ``Record | Closed`` only. ``with_superseded`` adds ``Superseded`` items, +computed from the ``Producer`` epoch that each Record carries, with no Store facts. +""" + +from collections.abc import AsyncGenerator + +from temporalio.streams import StreamOffset, _reader +from temporalio.streams._items import Closed, Record +from temporalio.streams._reader import Item +from temporalio.streams.superseded import Superseded, with_superseded +from temporalio.streams.system import models + + +def _ts(seconds: int) -> models.Timestamp: + return models.Timestamp(seconds=seconds, nanos=0) + + +def _activity( + id: str, run: str, attempt: int, seconds: int, standalone: bool = False +) -> models.Producer: + producer = models.ActivityProducer( + run_id=run, + epoch=models.ActivityEpoch( + attempt_scheduled_time=_ts(seconds), attempt=attempt + ), + ) + kind = ( + models.ProducerKindStandaloneActivity(producer) + if standalone + else models.ProducerKindActivity(producer) + ) + return models.Producer(id=id, kind=kind) + + +def _workflow(run: str) -> models.Producer: + return models.Producer( + id=f"wf:{run}", + kind=models.ProducerKindWorkflow( + models.WorkflowProducer( + run_id=run, epoch=models.WorkflowTaskEpoch(attempt=1) + ) + ), + ) + + +def _local_activity(id: str, run: str, attempt: int) -> models.Producer: + return models.Producer( + id=id, + kind=models.ProducerKindLocalActivity( + models.LocalActivityProducer( + run_id=run, + epoch=models.LocalActivityEpoch( + task=models.WorkflowTaskEpoch(attempt=attempt), attempt=attempt + ), + ) + ), + ) + + +def _client(attempt: int) -> models.Producer: + return models.Producer( + id="client:me", + kind=models.ProducerKindClient( + models.ClientProducer(epoch=models.ClientEpoch(attempt=attempt)) + ), + ) + + +def _record(offset: int, producer: models.Producer) -> Record[None]: + return Record(data=None, offset=StreamOffset._from_wire(offset), producer=producer) + + +async def _wrapped(*items: Item) -> list[Record[None] | Superseded | Closed]: + async def source() -> AsyncGenerator[Item, None]: + for item in items: + yield item + + reader: _reader.Reader[None] = _reader.Reader(source(), _reader.Position(0)) + return [item async for item in with_superseded(reader)] + + +async def test_an_activity_retry_supersedes_the_failed_attempt(): + first = _activity("act:wf:a:k", "r", attempt=1, seconds=100) + retry = _activity("act:wf:a:k", "r", attempt=2, seconds=101) + items = await _wrapped(_record(10, first), _record(11, first), _record(20, retry)) + [superseded] = [i for i in items if isinstance(i, Superseded)] + assert tuple(str(o) for o in superseded.withdrawn_offsets) == ("10", "11") + records = [i for i in items if isinstance(i, Record)] + assert [str(r.offset) for r in records] == ["10", "11", "20"] + # The Superseded item comes before the Record whose higher epoch withdrew the earlier ones. + assert str(items[2].offset) == "20" and isinstance(items[2], Superseded) + assert superseded.withdraws(_record(10, first)) + assert not superseded.withdraws(_record(20, retry)) + + +async def test_a_standalone_activity_retry_supersedes_the_failed_attempt(): + first = _activity("sa:a:r", "r", attempt=1, seconds=100, standalone=True) + retry = _activity("sa:a:r", "r", attempt=2, seconds=101, standalone=True) + items = await _wrapped(_record(10, first), _record(20, retry)) + [superseded] = [i for i in items if isinstance(i, Superseded)] + assert tuple(str(o) for o in superseded.withdrawn_offsets) == ("10",) + + +async def test_a_client_producer_never_supersedes(): + # A newer client epoch only fences, so a client's Records are never withdrawn. + items = await _wrapped(_record(10, _client(1)), _record(20, _client(2))) + assert not any(isinstance(i, Superseded) for i in items) + # withdraws() also never matches a client's Records. + synthetic = Superseded( + offset=StreamOffset.parse("20"), + producer=_activity("act:x", "r", attempt=2, seconds=101), + ) + assert not synthetic.withdraws(_record(10, _client(1))) + + +async def test_a_workflow_producer_never_supersedes(): + # A run's Producer id is stable across its Workflow Tasks, so a failed task's Records stay + # visible. + run = _workflow("run-1") + items = await _wrapped(_record(10, run), _record(20, run)) + assert not any(isinstance(i, Superseded) for i in items) + + +async def test_a_local_activity_producer_never_supersedes(): + first = _local_activity("la:run-1:k", "run-1", attempt=1) + retry = _local_activity("la:run-1:k", "run-1", attempt=2) + items = await _wrapped(_record(10, first), _record(20, retry)) + assert not any(isinstance(i, Superseded) for i in items) + + +async def test_withdraws_checks_a_record_kept_from_an_earlier_read(): + first = _activity("act:wf:a:k", "r", attempt=1, seconds=100) + retry = _activity("act:wf:a:k", "r", attempt=2, seconds=101) + live = await _wrapped(_record(10, first), _record(20, retry)) + [superseded] = [i for i in live if isinstance(i, Superseded)] + # An app that kept the failed attempt's Record from before a reload checks it, by the Record + # or by its Producer. + assert superseded.withdraws(_record(10, first)) + assert superseded.withdraws(first) + assert not superseded.withdraws(_record(20, retry)) + other = _activity("act:wf:b:k", "r2", attempt=1, seconds=100) + assert not superseded.withdraws(_record(99, other)) diff --git a/tests/contrib/streams/test_vectors.py b/tests/contrib/streams/test_vectors.py index e73253f2c9..e35cae8c94 100644 --- a/tests/contrib/streams/test_vectors.py +++ b/tests/contrib/streams/test_vectors.py @@ -1,8 +1,7 @@ from pathlib import Path from typing import Any -from temporalio.streams import _ids, _items -from temporalio.streams.system import models +from temporalio.streams import _ids from tests.contrib.streams._support import crate_file, vectors _SCRIPTS = Path(__file__).parents[3] / "scripts" / "_proto" @@ -45,80 +44,3 @@ def test_producer_ids_match_the_crate(): ) named = producers["named_client"] assert _ids.named_client_producer_id(named["name"]) == named["id"] - - -def _time(value: list[int] | None) -> models.Timestamp | None: - return models.Timestamp(seconds=value[0], nanos=value[1]) if value else None - - -def _task(value: dict[str, Any], prefix: str = "") -> models.WorkflowTaskEpoch: - return models.WorkflowTaskEpoch( - started_event_id=value.get(f"{prefix}started_event_id"), - attempt=value.get(f"{prefix}attempt"), - started_time=_time(value.get(f"{prefix}started_time")), - ) - - -def _epoch(kind: str, value: dict[str, Any] | None) -> models.ProducerEpoch | None: - # The vectors keep the flat fields of every kind, so the case comes from the kind. - if value is None: - return None - activity = models.ActivityEpoch( - attempt_scheduled_time=_time(value.get("attempt_scheduled_time")), - attempt=value.get("attempt"), - ) - cases: dict[str, models.ProducerEpochKind] = { - "workflow": models.ProducerEpochKindWorkflow(_task(value, "task_")), - "local_activity": models.ProducerEpochKindLocalActivity( - models.LocalActivityEpoch( - task=_task(value, "task_"), attempt=value.get("attempt") - ) - ), - "activity": models.ProducerEpochKindActivity(activity), - "standalone_activity": models.ProducerEpochKindStandaloneActivity(activity), - "client": models.ProducerEpochKindClient( - models.ClientEpoch(attempt=value.get("attempt")) - ), - } - return models.ProducerEpoch(kind=cases[kind]) - - -def test_withdrawals_match_the_crate(): - cases: Any = vectors("withdrawals.json") - assert cases - for case in cases: - record = case["record"]["producer"] - producer = models.Producer( - id=record["id"], - run_id=record.get("run_id"), - epoch=_epoch(record["kind"], record.get("epoch")), - ) - given = case["superseded"] - completed = given.get("previous_completed_task") - writer = models.StreamWriter( - producer=models.Producer( - id=given["producer_id"], - epoch=_epoch(given["producer_kind"], given.get("epoch")), - run_id=given.get("run_id"), - ), - previous_completed_task=_task(completed) if completed else None, - failed_tasks=given.get("failed_tasks", []), - withdraw_run=given.get("withdraw_run"), - ) - facts = models.StreamSuperseded(writer=writer) - assert _items.withdraws(facts, producer) == case["withdraws"], case["name"] - - -def test_epoch_order_matches_the_crate(): - cases: Any = vectors("epochs.json") - keys: list[tuple[tuple[Any, ...], bytes]] = [] - for case in cases: - epoch = _epoch(case["kind"], case["epoch"]) if case["kind"] != "empty" else None - python = _items._epoch_key(epoch or models.ProducerEpoch()) - keys.append((python, bytes.fromhex(case["key"]))) - assert len(keys) > 1 - # A byte compare of the crate's keys orders the epochs, so the Python keys compare the same. - for python_a, crate_a in keys: - for python_b, crate_b in keys: - assert (python_a < python_b) == (crate_a < crate_b) - assert (python_a == python_b) == (crate_a == crate_b) diff --git a/tests/contrib/streams/test_workflow_module.py b/tests/contrib/streams/test_workflow_module.py index 718797e4c7..88b77e9c7b 100644 --- a/tests/contrib/streams/test_workflow_module.py +++ b/tests/contrib/streams/test_workflow_module.py @@ -179,7 +179,6 @@ async def run(self) -> list[str]: tokens = await workflow.create_stream(own_id(), payload_type=str) generated = generated_stream( id=tokens.ref.id, - owner=tokens.ref.owner, backend=tokens.ref.backend or None, body_type=str, ) @@ -207,43 +206,6 @@ async def test_generated_workflow_calls_run_as_local_activities(stream_client: C assert EventType.EVENT_TYPE_NEXUS_OPERATION_SCHEDULED not in events -@workflow.defn -class RefusedCalls: - @workflow.run - async def run(self) -> list[str]: - tokens = await workflow.create_stream(own_id(), payload_type=str) - refused = [] - # A Stream bound to another run chain fails before the block runs, and before any call. - owner = streams.Execution( - type=streams.ExecutionType.ExecutionTypeWorkflow, - business_id="other", - run_id="run-1", - ) - other = workflow.get_stream(streams.StreamRef[str](id="other", owner=owner)) - try: - async with other: - refused.append("entered") - except streams.StreamError as err: - refused.append(reason(err)) - try: - await other.close() - except streams.StreamError as err: - refused.append(reason(err)) - await tokens.close() - return refused - - -async def test_calls_that_workflow_code_cant_make_raise(stream_client: Client): - async with new_worker(stream_client, RefusedCalls) as worker: - result = await stream_client.execute_workflow( - RefusedCalls.run, id=f"wf-{uuid.uuid4()}", task_queue=worker.task_queue - ) - assert result == [ - "STREAM_ERROR_REASON_NOT_OWNER", - "STREAM_ERROR_REASON_NOT_OWNER", - ] - - @workflow.defn class Blocks: @workflow.run @@ -481,10 +443,8 @@ async def test_block_end_leaves_the_stream_open_for_a_retry_run(stream_client: C # Only the block end of the run that succeeds closes the Stream. assert await status(handle) == CLOSED written = [item for item in items if isinstance(item, streams.Record)] - superseded = [item for item in items if isinstance(item, streams.Superseded)] + # Both runs' Records stay visible, since a run's writes are never withdrawn. assert [r.data for r in written] == ["run-1", "run-2"] - # The retry run's first write withdraws the Records of the run that it retries. - assert [r.data for r in written if superseded[0].withdraws(r)] == ["run-1"] @workflow.defn @@ -672,12 +632,8 @@ async def test_failed_task_writes_no_close(stream_client: Client): items = await read(handle) assert await status(handle) == CLOSED written = [item for item in items if isinstance(item, streams.Record)] - superseded = [item for item in items if isinstance(item, streams.Superseded)] - # The retried task writes the token again, and withdraws the failed task's copy. + # The retried task writes the token again, and both copies stay visible. assert [r.data for r in written] == ["token", "token"] - assert [r for r in written if any(s.withdraws(r) for s in superseded)] == [ - written[0] - ] history = await handle.fetch_history() # The failed task's close was never sent, so History has one close Activity. scheduled = [ @@ -1179,39 +1135,37 @@ async def run(self, backend: str) -> str: @workflow.defn -class CreatesAfterAnIdleTime: +class CreatesThenCloses: @workflow.run - async def run(self) -> str: - await workflow.create_stream(own_id(), payload_type=str) - await workflow.sleep(timedelta(seconds=1.5)) - try: - await workflow.create_stream( - own_id(), payload_type=str, id_conflict_policy=USE_EXISTING - ) - except streams.StreamError as err: - if err.reason == StreamErrorReason.STREAM_ERROR_REASON_EXPIRED: - return reason(err) - raise - return "created" + async def run(self) -> None: + stream = await workflow.create_stream(own_id(), payload_type=str) + await stream.append("token") + await stream.close() async def test_create_of_an_expired_workflow_stream_raises(client: Client): # Retention is a backend setting now. A Workflow's standalone Stream expires one Retention - # after its last write. + # after its close, since safety_ttl keeps it open. A later client create finds it expired. config = client.config() config["stream_backends"] = [ MemoryStreamBackend(retention=timedelta(milliseconds=500), default=True) ] memory_client = Client(**config) - async with new_worker(memory_client, CreatesAfterAnIdleTime) as worker: - assert ( - await memory_client.execute_workflow( - CreatesAfterAnIdleTime.run, - id=f"wf-{uuid.uuid4()}", - task_queue=worker.task_queue, - ) - == "STREAM_ERROR_REASON_EXPIRED" + async with new_worker(memory_client, CreatesThenCloses) as worker: + handle = await memory_client.start_workflow( + CreatesThenCloses.run, + id=f"wf-{uuid.uuid4()}", + task_queue=worker.task_queue, ) + await handle.result() + await asyncio.sleep(1.5) + with pytest.raises(streams.StreamError) as expired: + await memory_client.create_stream( + stream_id(handle), + payload_type=str, + id_conflict_policy=USE_EXISTING, + ) + assert expired.value.reason == StreamErrorReason.STREAM_ERROR_REASON_EXPIRED async def test_default_is_native_with_no_default_backend(client: Client): From efb95d48b5b62e72a3b940e6067da9f1d1d6d733 Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 8 Nov 2026 19:55:46 -0700 Subject: [PATCH 6/7] Dropped the OWNER_* close statuses and trimmed the Stream close. --- scripts/_wit/stream-service.wit | 14 +---- temporalio/api/enums/v1/stream_pb2.py | 24 ++++----- temporalio/api/enums/v1/stream_pb2.pyi | 33 +++--------- temporalio/api/stream/v1/message_pb2.py | 69 ++++++++++++------------ temporalio/api/stream/v1/message_pb2.pyi | 33 +++--------- temporalio/bridge/sdk-core | 2 +- temporalio/streams/_close.py | 20 +------ temporalio/streams/_items.py | 6 +-- temporalio/streams/_reader.py | 1 - temporalio/streams/system/models.py | 18 ------- tests/contrib/streams/test_refs.py | 5 +- 11 files changed, 63 insertions(+), 162 deletions(-) diff --git a/scripts/_wit/stream-service.wit b/scripts/_wit/stream-service.wit index c7944a0b01..c3822057ce 100644 --- a/scripts/_wit/stream-service.wit +++ b/scripts/_wit/stream-service.wit @@ -14,8 +14,6 @@ world system { interface stream-service { use nexus:temporal-types/model@1.0.0.{ duration, - failure, - payload, placeholder, }; @@ -142,18 +140,12 @@ interface stream-service { producer: option, } - /// How a Stream closed. Only the owner's end on the server terminates or times out. + /// How a Stream closed. A Producer's close gives CLOSED. Native Streams add their own later. /// @nexus.proto "temporal.api.enums.v1.StreamCloseStatus" enum stream-close-status { /// @nexus.fallback stream-close-status-unspecified, stream-close-status-closed, - stream-close-status-owner-completed, - stream-close-status-owner-failed, - stream-close-status-owner-canceled, - stream-close-status-owner-terminated, - stream-close-status-owner-timed-out, - stream-close-status-idle, } /// @nexus.doc "The close of a Stream. A read gives it last." @@ -161,9 +153,7 @@ interface stream-service { record stream-close { /// @nexus.doc "The Producer that closed the Stream." producer: option, - %result: option, - failure: option, - /// @nexus.doc "Which of result or failure is set, and why the Stream closed." + /// @nexus.doc "Why the Stream closed." status: option, } diff --git a/temporalio/api/enums/v1/stream_pb2.py b/temporalio/api/enums/v1/stream_pb2.py index 143b92ac59..55cebd6475 100644 --- a/temporalio/api/enums/v1/stream_pb2.py +++ b/temporalio/api/enums/v1/stream_pb2.py @@ -16,7 +16,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n"temporal/api/enums/v1/stream.proto\x12\x15temporal.api.enums.v1*z\n\x0cStreamStatus\x12\x1d\n\x19STREAM_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12STREAM_STATUS_OPEN\x10\x01\x12\x18\n\x14STREAM_STATUS_CLOSED\x10\x02\x12\x19\n\x15STREAM_STATUS_EXPIRED\x10\x03*\xc0\x02\n\x11StreamCloseStatus\x12#\n\x1fSTREAM_CLOSE_STATUS_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTREAM_CLOSE_STATUS_CLOSED\x10\x01\x12\'\n#STREAM_CLOSE_STATUS_OWNER_COMPLETED\x10\x02\x12$\n STREAM_CLOSE_STATUS_OWNER_FAILED\x10\x03\x12&\n"STREAM_CLOSE_STATUS_OWNER_CANCELED\x10\x04\x12(\n$STREAM_CLOSE_STATUS_OWNER_TERMINATED\x10\x05\x12\'\n#STREAM_CLOSE_STATUS_OWNER_TIMED_OUT\x10\x06\x12\x1c\n\x18STREAM_CLOSE_STATUS_IDLE\x10\x07*p\n\x0fStreamReadStart\x12!\n\x1dSTREAM_READ_START_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSTREAM_READ_START_BEGINNING\x10\x01\x12\x19\n\x15STREAM_READ_START_END\x10\x02*\x93\x01\n\x16StreamIdConflictPolicy\x12)\n%STREAM_ID_CONFLICT_POLICY_UNSPECIFIED\x10\x00\x12"\n\x1eSTREAM_ID_CONFLICT_POLICY_FAIL\x10\x01\x12*\n&STREAM_ID_CONFLICT_POLICY_USE_EXISTING\x10\x02*\x90\x05\n\x11StreamErrorReason\x12#\n\x1fSTREAM_ERROR_REASON_UNSPECIFIED\x10\x00\x12!\n\x1dSTREAM_ERROR_REASON_NOT_FOUND\x10\x01\x12\x1f\n\x1bSTREAM_ERROR_REASON_EXPIRED\x10\x02\x12\x1e\n\x1aSTREAM_ERROR_REASON_CLOSED\x10\x03\x12\x1e\n\x1aSTREAM_ERROR_REASON_FENCED\x10\x04\x12(\n$STREAM_ERROR_REASON_SEQUENCE_TOO_LOW\x10\x05\x12%\n!STREAM_ERROR_REASON_SEQ_DIVERGENT\x10\x06\x12!\n\x1dSTREAM_ERROR_REASON_NOT_OWNER\x10\x07\x12&\n"STREAM_ERROR_REASON_FORMAT_TOO_NEW\x10\x08\x12\'\n#STREAM_ERROR_REASON_OFFSET_TOO_HIGH\x10\t\x12!\n\x1dSTREAM_ERROR_REASON_TOO_LARGE\x10\n\x12\x1f\n\x1bSTREAM_ERROR_REASON_INVALID\x10\x0b\x12+\n\'STREAM_ERROR_REASON_STORAGE_UNAVAILABLE\x10\x0c\x12&\n"STREAM_ERROR_REASON_UNKNOWN_WRITER\x10\r\x12#\n\x1fSTREAM_ERROR_REASON_UNSUPPORTED\x10\x0e\x12&\n"STREAM_ERROR_REASON_ALREADY_EXISTS\x10\x0f\x12\'\n#STREAM_ERROR_REASON_STORAGE_CORRUPT\x10\x10\x42\x83\x01\n\x18io.temporal.api.enums.v1B\x0bStreamProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3' + b'\n"temporal/api/enums/v1/stream.proto\x12\x15temporal.api.enums.v1*z\n\x0cStreamStatus\x12\x1d\n\x19STREAM_STATUS_UNSPECIFIED\x10\x00\x12\x16\n\x12STREAM_STATUS_OPEN\x10\x01\x12\x18\n\x14STREAM_STATUS_CLOSED\x10\x02\x12\x19\n\x15STREAM_STATUS_EXPIRED\x10\x03*X\n\x11StreamCloseStatus\x12#\n\x1fSTREAM_CLOSE_STATUS_UNSPECIFIED\x10\x00\x12\x1e\n\x1aSTREAM_CLOSE_STATUS_CLOSED\x10\x01*p\n\x0fStreamReadStart\x12!\n\x1dSTREAM_READ_START_UNSPECIFIED\x10\x00\x12\x1f\n\x1bSTREAM_READ_START_BEGINNING\x10\x01\x12\x19\n\x15STREAM_READ_START_END\x10\x02*\x93\x01\n\x16StreamIdConflictPolicy\x12)\n%STREAM_ID_CONFLICT_POLICY_UNSPECIFIED\x10\x00\x12"\n\x1eSTREAM_ID_CONFLICT_POLICY_FAIL\x10\x01\x12*\n&STREAM_ID_CONFLICT_POLICY_USE_EXISTING\x10\x02*\x90\x05\n\x11StreamErrorReason\x12#\n\x1fSTREAM_ERROR_REASON_UNSPECIFIED\x10\x00\x12!\n\x1dSTREAM_ERROR_REASON_NOT_FOUND\x10\x01\x12\x1f\n\x1bSTREAM_ERROR_REASON_EXPIRED\x10\x02\x12\x1e\n\x1aSTREAM_ERROR_REASON_CLOSED\x10\x03\x12\x1e\n\x1aSTREAM_ERROR_REASON_FENCED\x10\x04\x12(\n$STREAM_ERROR_REASON_SEQUENCE_TOO_LOW\x10\x05\x12%\n!STREAM_ERROR_REASON_SEQ_DIVERGENT\x10\x06\x12!\n\x1dSTREAM_ERROR_REASON_NOT_OWNER\x10\x07\x12&\n"STREAM_ERROR_REASON_FORMAT_TOO_NEW\x10\x08\x12\'\n#STREAM_ERROR_REASON_OFFSET_TOO_HIGH\x10\t\x12!\n\x1dSTREAM_ERROR_REASON_TOO_LARGE\x10\n\x12\x1f\n\x1bSTREAM_ERROR_REASON_INVALID\x10\x0b\x12+\n\'STREAM_ERROR_REASON_STORAGE_UNAVAILABLE\x10\x0c\x12&\n"STREAM_ERROR_REASON_UNKNOWN_WRITER\x10\r\x12#\n\x1fSTREAM_ERROR_REASON_UNSUPPORTED\x10\x0e\x12&\n"STREAM_ERROR_REASON_ALREADY_EXISTS\x10\x0f\x12\'\n#STREAM_ERROR_REASON_STORAGE_CORRUPT\x10\x10\x42\x83\x01\n\x18io.temporal.api.enums.v1B\x0bStreamProtoP\x01Z!go.temporal.io/api/enums/v1;enums\xaa\x02\x17Temporalio.Api.Enums.V1\xea\x02\x1aTemporalio::Api::Enums::V1b\x06proto3' ) _STREAMSTATUS = DESCRIPTOR.enum_types_by_name["StreamStatus"] @@ -35,12 +35,6 @@ STREAM_STATUS_EXPIRED = 3 STREAM_CLOSE_STATUS_UNSPECIFIED = 0 STREAM_CLOSE_STATUS_CLOSED = 1 -STREAM_CLOSE_STATUS_OWNER_COMPLETED = 2 -STREAM_CLOSE_STATUS_OWNER_FAILED = 3 -STREAM_CLOSE_STATUS_OWNER_CANCELED = 4 -STREAM_CLOSE_STATUS_OWNER_TERMINATED = 5 -STREAM_CLOSE_STATUS_OWNER_TIMED_OUT = 6 -STREAM_CLOSE_STATUS_IDLE = 7 STREAM_READ_START_UNSPECIFIED = 0 STREAM_READ_START_BEGINNING = 1 STREAM_READ_START_END = 2 @@ -71,12 +65,12 @@ DESCRIPTOR._serialized_options = b"\n\030io.temporal.api.enums.v1B\013StreamProtoP\001Z!go.temporal.io/api/enums/v1;enums\252\002\027Temporalio.Api.Enums.V1\352\002\032Temporalio::Api::Enums::V1" _STREAMSTATUS._serialized_start = 61 _STREAMSTATUS._serialized_end = 183 - _STREAMCLOSESTATUS._serialized_start = 186 - _STREAMCLOSESTATUS._serialized_end = 506 - _STREAMREADSTART._serialized_start = 508 - _STREAMREADSTART._serialized_end = 620 - _STREAMIDCONFLICTPOLICY._serialized_start = 623 - _STREAMIDCONFLICTPOLICY._serialized_end = 770 - _STREAMERRORREASON._serialized_start = 773 - _STREAMERRORREASON._serialized_end = 1429 + _STREAMCLOSESTATUS._serialized_start = 185 + _STREAMCLOSESTATUS._serialized_end = 273 + _STREAMREADSTART._serialized_start = 275 + _STREAMREADSTART._serialized_end = 387 + _STREAMIDCONFLICTPOLICY._serialized_start = 390 + _STREAMIDCONFLICTPOLICY._serialized_end = 537 + _STREAMERRORREASON._serialized_start = 540 + _STREAMERRORREASON._serialized_end = 1196 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/enums/v1/stream_pb2.pyi b/temporalio/api/enums/v1/stream_pb2.pyi index 2d026f71c8..d3e7152efb 100644 --- a/temporalio/api/enums/v1/stream_pb2.pyi +++ b/temporalio/api/enums/v1/stream_pb2.pyi @@ -59,32 +59,19 @@ class _StreamCloseStatusEnumTypeWrapper( DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor STREAM_CLOSE_STATUS_UNSPECIFIED: _StreamCloseStatus.ValueType # 0 STREAM_CLOSE_STATUS_CLOSED: _StreamCloseStatus.ValueType # 1 - STREAM_CLOSE_STATUS_OWNER_COMPLETED: _StreamCloseStatus.ValueType # 2 - STREAM_CLOSE_STATUS_OWNER_FAILED: _StreamCloseStatus.ValueType # 3 - STREAM_CLOSE_STATUS_OWNER_CANCELED: _StreamCloseStatus.ValueType # 4 - STREAM_CLOSE_STATUS_OWNER_TERMINATED: _StreamCloseStatus.ValueType # 5 - STREAM_CLOSE_STATUS_OWNER_TIMED_OUT: _StreamCloseStatus.ValueType # 6 - STREAM_CLOSE_STATUS_IDLE: _StreamCloseStatus.ValueType # 7 class StreamCloseStatus( _StreamCloseStatus, metaclass=_StreamCloseStatusEnumTypeWrapper ): """Experimental. Might change in the future. - How a Stream closed. CLOSED comes from a Producer that called close(). The OWNER_ values and - IDLE come from the server (native Streams) when the owner ends or the Stream goes idle. A Brook - SDK store only emits CLOSED. + How a Stream closed. CLOSED comes from a Producer that called close(). Native Streams add their + own close statuses later. (-- api-linter: core::0216::synonyms=disabled aip.dev/not-precedent: Named consistently with WorkflowExecutionStatus. --) """ STREAM_CLOSE_STATUS_UNSPECIFIED: StreamCloseStatus.ValueType # 0 STREAM_CLOSE_STATUS_CLOSED: StreamCloseStatus.ValueType # 1 -STREAM_CLOSE_STATUS_OWNER_COMPLETED: StreamCloseStatus.ValueType # 2 -STREAM_CLOSE_STATUS_OWNER_FAILED: StreamCloseStatus.ValueType # 3 -STREAM_CLOSE_STATUS_OWNER_CANCELED: StreamCloseStatus.ValueType # 4 -STREAM_CLOSE_STATUS_OWNER_TERMINATED: StreamCloseStatus.ValueType # 5 -STREAM_CLOSE_STATUS_OWNER_TIMED_OUT: StreamCloseStatus.ValueType # 6 -STREAM_CLOSE_STATUS_IDLE: StreamCloseStatus.ValueType # 7 global___StreamCloseStatus = StreamCloseStatus class _StreamReadStart: @@ -162,9 +149,7 @@ class _StreamErrorReasonEnumTypeWrapper( STREAM_ERROR_REASON_CLOSED: _StreamErrorReason.ValueType # 3 """The Stream is closed. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_FENCED: _StreamErrorReason.ValueType # 4 - """The Producer's epoch is lower than the newest one, its Workflow Task was withdrawn, or a - Workflow retry withdrew its run. FAILED_PRECONDITION. - """ + """The Producer's epoch is lower than the newest one. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_SEQUENCE_TOO_LOW: _StreamErrorReason.ValueType # 5 """The batch has a lower `sequence` than the last one of its epoch. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_SEQ_DIVERGENT: _StreamErrorReason.ValueType # 6 @@ -172,9 +157,7 @@ class _StreamErrorReasonEnumTypeWrapper( FAILED_PRECONDITION. """ STREAM_ERROR_REASON_NOT_OWNER: _StreamErrorReason.ValueType # 7 - """A create gives another owner than the Stream has, or the owner doesn't allow this close. - FAILED_PRECONDITION. - """ + """The caller may not create or close this Stream. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_FORMAT_TOO_NEW: _StreamErrorReason.ValueType # 8 """A newer format version wrote the Stream. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_OFFSET_TOO_HIGH: _StreamErrorReason.ValueType # 9 @@ -214,9 +197,7 @@ STREAM_ERROR_REASON_EXPIRED: StreamErrorReason.ValueType # 2 STREAM_ERROR_REASON_CLOSED: StreamErrorReason.ValueType # 3 """The Stream is closed. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_FENCED: StreamErrorReason.ValueType # 4 -"""The Producer's epoch is lower than the newest one, its Workflow Task was withdrawn, or a -Workflow retry withdrew its run. FAILED_PRECONDITION. -""" +"""The Producer's epoch is lower than the newest one. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_SEQUENCE_TOO_LOW: StreamErrorReason.ValueType # 5 """The batch has a lower `sequence` than the last one of its epoch. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_SEQ_DIVERGENT: StreamErrorReason.ValueType # 6 @@ -224,9 +205,7 @@ STREAM_ERROR_REASON_SEQ_DIVERGENT: StreamErrorReason.ValueType # 6 FAILED_PRECONDITION. """ STREAM_ERROR_REASON_NOT_OWNER: StreamErrorReason.ValueType # 7 -"""A create gives another owner than the Stream has, or the owner doesn't allow this close. -FAILED_PRECONDITION. -""" +"""The caller may not create or close this Stream. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_FORMAT_TOO_NEW: StreamErrorReason.ValueType # 8 """A newer format version wrote the Stream. FAILED_PRECONDITION.""" STREAM_ERROR_REASON_OFFSET_TOO_HIGH: StreamErrorReason.ValueType # 9 diff --git a/temporalio/api/stream/v1/message_pb2.py b/temporalio/api/stream/v1/message_pb2.py index 846e89a5ac..d06834979c 100644 --- a/temporalio/api/stream/v1/message_pb2.py +++ b/temporalio/api/stream/v1/message_pb2.py @@ -22,12 +22,9 @@ from temporalio.api.enums.v1 import ( stream_pb2 as temporal_dot_api_dot_enums_dot_v1_dot_stream__pb2, ) -from temporalio.api.failure.v1 import ( - message_pb2 as temporal_dot_api_dot_failure_dot_v1_dot_message__pb2, -) DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n$temporal/api/stream/v1/message.proto\x12\x16temporal.api.stream.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a"temporal/api/enums/v1/stream.proto\x1a%temporal/api/failure/v1/message.proto"(\n\tStreamRef\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07\x62\x61\x63kend\x18\x03 \x01(\t"p\n\x11WorkflowTaskEpoch\x12\x18\n\x10started_event_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05\x12\x30\n\x0cstarted_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"^\n\x12LocalActivityEpoch\x12\x37\n\x04task\x18\x01 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05"\\\n\rActivityEpoch\x12:\n\x16\x61ttempt_scheduled_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05"\x1e\n\x0b\x43lientEpoch\x12\x0f\n\x07\x61ttempt\x18\x01 \x01(\x05"\\\n\x10WorkflowProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x38\n\x05\x65poch\x18\x02 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch"b\n\x15LocalActivityProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x39\n\x05\x65poch\x18\x02 \x01(\x0b\x32*.temporal.api.stream.v1.LocalActivityEpoch"X\n\x10\x41\x63tivityProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x34\n\x05\x65poch\x18\x02 \x01(\x0b\x32%.temporal.api.stream.v1.ActivityEpoch"D\n\x0e\x43lientProducer\x12\x32\n\x05\x65poch\x18\x01 \x01(\x0b\x32#.temporal.api.stream.v1.ClientEpoch"\xe6\x02\n\x08Producer\x12\n\n\x02id\x18\x01 \x01(\t\x12<\n\x08workflow\x18\x02 \x01(\x0b\x32(.temporal.api.stream.v1.WorkflowProducerH\x00\x12G\n\x0elocal_activity\x18\x03 \x01(\x0b\x32-.temporal.api.stream.v1.LocalActivityProducerH\x00\x12<\n\x08\x61\x63tivity\x18\x04 \x01(\x0b\x32(.temporal.api.stream.v1.ActivityProducerH\x00\x12G\n\x13standalone_activity\x18\x05 \x01(\x0b\x32(.temporal.api.stream.v1.ActivityProducerH\x00\x12\x38\n\x06\x63lient\x18\x06 \x01(\x0b\x32&.temporal.api.stream.v1.ClientProducerH\x00\x42\x06\n\x04kind"B\n\x0cStreamWriter\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer"=\n\x0c\x41ppendRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"o\n\nStreamData\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12-\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"\xdf\x01\n\x0bStreamClose\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12/\n\x06result\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload\x12\x31\n\x07\x66\x61ilure\x18\x03 \x01(\x0b\x32 .temporal.api.failure.v1.Failure\x12\x38\n\x06status\x18\x04 \x01(\x0e\x32(.temporal.api.enums.v1.StreamCloseStatus"\x90\x01\n\x0cStreamRecord\x12\x0e\n\x06offset\x18\x01 \x01(\x03\x12\x32\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32".temporal.api.stream.v1.StreamDataH\x00\x12\x34\n\x05\x63lose\x18\x04 \x01(\x0b\x32#.temporal.api.stream.v1.StreamCloseH\x00\x42\x06\n\x04item"N\n\x12StreamErrorDetails\x12\x38\n\x06reason\x18\x01 \x01(\x0e\x32(.temporal.api.enums.v1.StreamErrorReasonB\x89\x01\n\x19io.temporal.api.stream.v1B\x0cMessageProtoP\x01Z#go.temporal.io/api/stream/v1;stream\xaa\x02\x18Temporalio.Api.Stream.V1\xea\x02\x1bTemporalio::Api::Stream::V1b\x06proto3' + b'\n$temporal/api/stream/v1/message.proto\x12\x16temporal.api.stream.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a$temporal/api/common/v1/message.proto\x1a"temporal/api/enums/v1/stream.proto"(\n\tStreamRef\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0f\n\x07\x62\x61\x63kend\x18\x03 \x01(\t"p\n\x11WorkflowTaskEpoch\x12\x18\n\x10started_event_id\x18\x01 \x01(\x03\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05\x12\x30\n\x0cstarted_time\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp"^\n\x12LocalActivityEpoch\x12\x37\n\x04task\x18\x01 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05"\\\n\rActivityEpoch\x12:\n\x16\x61ttempt_scheduled_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0f\n\x07\x61ttempt\x18\x02 \x01(\x05"\x1e\n\x0b\x43lientEpoch\x12\x0f\n\x07\x61ttempt\x18\x01 \x01(\x05"\\\n\x10WorkflowProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x38\n\x05\x65poch\x18\x02 \x01(\x0b\x32).temporal.api.stream.v1.WorkflowTaskEpoch"b\n\x15LocalActivityProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x39\n\x05\x65poch\x18\x02 \x01(\x0b\x32*.temporal.api.stream.v1.LocalActivityEpoch"X\n\x10\x41\x63tivityProducer\x12\x0e\n\x06run_id\x18\x01 \x01(\t\x12\x34\n\x05\x65poch\x18\x02 \x01(\x0b\x32%.temporal.api.stream.v1.ActivityEpoch"D\n\x0e\x43lientProducer\x12\x32\n\x05\x65poch\x18\x01 \x01(\x0b\x32#.temporal.api.stream.v1.ClientEpoch"\xe6\x02\n\x08Producer\x12\n\n\x02id\x18\x01 \x01(\t\x12<\n\x08workflow\x18\x02 \x01(\x0b\x32(.temporal.api.stream.v1.WorkflowProducerH\x00\x12G\n\x0elocal_activity\x18\x03 \x01(\x0b\x32-.temporal.api.stream.v1.LocalActivityProducerH\x00\x12<\n\x08\x61\x63tivity\x18\x04 \x01(\x0b\x32(.temporal.api.stream.v1.ActivityProducerH\x00\x12G\n\x13standalone_activity\x18\x05 \x01(\x0b\x32(.temporal.api.stream.v1.ActivityProducerH\x00\x12\x38\n\x06\x63lient\x18\x06 \x01(\x0b\x32&.temporal.api.stream.v1.ClientProducerH\x00\x42\x06\n\x04kind"B\n\x0cStreamWriter\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer"=\n\x0c\x41ppendRecord\x12-\n\x04\x62ody\x18\x01 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"o\n\nStreamData\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12-\n\x04\x62ody\x18\x02 \x01(\x0b\x32\x1f.temporal.api.common.v1.Payload"\x98\x01\n\x0bStreamClose\x12\x32\n\x08producer\x18\x01 \x01(\x0b\x32 .temporal.api.stream.v1.Producer\x12\x38\n\x06status\x18\x04 \x01(\x0e\x32(.temporal.api.enums.v1.StreamCloseStatusJ\x04\x08\x02\x10\x03J\x04\x08\x03\x10\x04R\x06resultR\x07\x66\x61ilure"\x90\x01\n\x0cStreamRecord\x12\x0e\n\x06offset\x18\x01 \x01(\x03\x12\x32\n\x04\x64\x61ta\x18\x02 \x01(\x0b\x32".temporal.api.stream.v1.StreamDataH\x00\x12\x34\n\x05\x63lose\x18\x04 \x01(\x0b\x32#.temporal.api.stream.v1.StreamCloseH\x00\x42\x06\n\x04item"N\n\x12StreamErrorDetails\x12\x38\n\x06reason\x18\x01 \x01(\x0e\x32(.temporal.api.enums.v1.StreamErrorReasonB\x89\x01\n\x19io.temporal.api.stream.v1B\x0cMessageProtoP\x01Z#go.temporal.io/api/stream/v1;stream\xaa\x02\x18Temporalio.Api.Stream.V1\xea\x02\x1bTemporalio::Api::Stream::V1b\x06proto3' ) @@ -226,36 +223,36 @@ if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b"\n\031io.temporal.api.stream.v1B\014MessageProtoP\001Z#go.temporal.io/api/stream/v1;stream\252\002\030Temporalio.Api.Stream.V1\352\002\033Temporalio::Api::Stream::V1" - _STREAMREF._serialized_start = 210 - _STREAMREF._serialized_end = 250 - _WORKFLOWTASKEPOCH._serialized_start = 252 - _WORKFLOWTASKEPOCH._serialized_end = 364 - _LOCALACTIVITYEPOCH._serialized_start = 366 - _LOCALACTIVITYEPOCH._serialized_end = 460 - _ACTIVITYEPOCH._serialized_start = 462 - _ACTIVITYEPOCH._serialized_end = 554 - _CLIENTEPOCH._serialized_start = 556 - _CLIENTEPOCH._serialized_end = 586 - _WORKFLOWPRODUCER._serialized_start = 588 - _WORKFLOWPRODUCER._serialized_end = 680 - _LOCALACTIVITYPRODUCER._serialized_start = 682 - _LOCALACTIVITYPRODUCER._serialized_end = 780 - _ACTIVITYPRODUCER._serialized_start = 782 - _ACTIVITYPRODUCER._serialized_end = 870 - _CLIENTPRODUCER._serialized_start = 872 - _CLIENTPRODUCER._serialized_end = 940 - _PRODUCER._serialized_start = 943 - _PRODUCER._serialized_end = 1301 - _STREAMWRITER._serialized_start = 1303 - _STREAMWRITER._serialized_end = 1369 - _APPENDRECORD._serialized_start = 1371 - _APPENDRECORD._serialized_end = 1432 - _STREAMDATA._serialized_start = 1434 - _STREAMDATA._serialized_end = 1545 - _STREAMCLOSE._serialized_start = 1548 - _STREAMCLOSE._serialized_end = 1771 - _STREAMRECORD._serialized_start = 1774 - _STREAMRECORD._serialized_end = 1918 - _STREAMERRORDETAILS._serialized_start = 1920 - _STREAMERRORDETAILS._serialized_end = 1998 + _STREAMREF._serialized_start = 171 + _STREAMREF._serialized_end = 211 + _WORKFLOWTASKEPOCH._serialized_start = 213 + _WORKFLOWTASKEPOCH._serialized_end = 325 + _LOCALACTIVITYEPOCH._serialized_start = 327 + _LOCALACTIVITYEPOCH._serialized_end = 421 + _ACTIVITYEPOCH._serialized_start = 423 + _ACTIVITYEPOCH._serialized_end = 515 + _CLIENTEPOCH._serialized_start = 517 + _CLIENTEPOCH._serialized_end = 547 + _WORKFLOWPRODUCER._serialized_start = 549 + _WORKFLOWPRODUCER._serialized_end = 641 + _LOCALACTIVITYPRODUCER._serialized_start = 643 + _LOCALACTIVITYPRODUCER._serialized_end = 741 + _ACTIVITYPRODUCER._serialized_start = 743 + _ACTIVITYPRODUCER._serialized_end = 831 + _CLIENTPRODUCER._serialized_start = 833 + _CLIENTPRODUCER._serialized_end = 901 + _PRODUCER._serialized_start = 904 + _PRODUCER._serialized_end = 1262 + _STREAMWRITER._serialized_start = 1264 + _STREAMWRITER._serialized_end = 1330 + _APPENDRECORD._serialized_start = 1332 + _APPENDRECORD._serialized_end = 1393 + _STREAMDATA._serialized_start = 1395 + _STREAMDATA._serialized_end = 1506 + _STREAMCLOSE._serialized_start = 1509 + _STREAMCLOSE._serialized_end = 1661 + _STREAMRECORD._serialized_start = 1664 + _STREAMRECORD._serialized_end = 1808 + _STREAMERRORDETAILS._serialized_start = 1810 + _STREAMERRORDETAILS._serialized_end = 1888 # @@protoc_insertion_point(module_scope) diff --git a/temporalio/api/stream/v1/message_pb2.pyi b/temporalio/api/stream/v1/message_pb2.pyi index 277bff996c..24fd9a82a0 100644 --- a/temporalio/api/stream/v1/message_pb2.pyi +++ b/temporalio/api/stream/v1/message_pb2.pyi @@ -12,7 +12,6 @@ import google.protobuf.timestamp_pb2 import temporalio.api.common.v1.message_pb2 import temporalio.api.enums.v1.stream_pb2 -import temporalio.api.failure.v1.message_pb2 if sys.version_info >= (3, 8): import typing as typing_extensions @@ -177,7 +176,7 @@ global___ClientEpoch = ClientEpoch class WorkflowProducer(google.protobuf.message.Message): """Experimental. Might change in the future. - A Workflow run Producer. It writes in its Workflow Task. A higher epoch withdraws the Records of + A Workflow run Producer. It writes in its Workflow Task. A higher epoch supersedes the Records of the run's Workflow Tasks that never completed, with those of their Local Activities. """ @@ -207,7 +206,7 @@ global___WorkflowProducer = WorkflowProducer class LocalActivityProducer(google.protobuf.message.Message): """Experimental. Might change in the future. - A Local Activity Producer. A higher epoch withdraws the Producer's lower epochs. + A Local Activity Producer. A higher epoch supersedes the Producer's lower epochs. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -236,7 +235,7 @@ global___LocalActivityProducer = LocalActivityProducer class ActivityProducer(google.protobuf.message.Message): """Experimental. Might change in the future. An Activity Producer, for an Activity of a Workflow or a Standalone Activity. A higher epoch - withdraws the Producer's lower epochs. + supersedes the Producer's lower epochs. """ DESCRIPTOR: google.protobuf.descriptor.Descriptor @@ -290,7 +289,7 @@ global___ClientProducer = ClientProducer class Producer(google.protobuf.message.Message): """Experimental. Might change in the future. The code that wrote a batch. Its id stays the same across attempts, and the kind's case orders - its epochs and picks which earlier Records a higher epoch withdraws. A run-based kind carries its + its epochs and picks which earlier Records a higher epoch supersedes. A run-based kind carries its run, so a client can't. """ @@ -459,43 +458,25 @@ class StreamClose(google.protobuf.message.Message): DESCRIPTOR: google.protobuf.descriptor.Descriptor PRODUCER_FIELD_NUMBER: builtins.int - RESULT_FIELD_NUMBER: builtins.int - FAILURE_FIELD_NUMBER: builtins.int STATUS_FIELD_NUMBER: builtins.int @property def producer(self) -> global___Producer: """The Producer that closed the Stream.""" - @property - def result(self) -> temporalio.api.common.v1.message_pb2.Payload: ... - @property - def failure(self) -> temporalio.api.failure.v1.message_pb2.Failure: ... status: temporalio.api.enums.v1.stream_pb2.StreamCloseStatus.ValueType - """Which of result or failure is set, and why the Stream closed.""" + """Why the Stream closed.""" def __init__( self, *, producer: global___Producer | None = ..., - result: temporalio.api.common.v1.message_pb2.Payload | None = ..., - failure: temporalio.api.failure.v1.message_pb2.Failure | None = ..., status: temporalio.api.enums.v1.stream_pb2.StreamCloseStatus.ValueType = ..., ) -> None: ... def HasField( - self, - field_name: typing_extensions.Literal[ - "failure", b"failure", "producer", b"producer", "result", b"result" - ], + self, field_name: typing_extensions.Literal["producer", b"producer"] ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal[ - "failure", - b"failure", - "producer", - b"producer", - "result", - b"result", - "status", - b"status", + "producer", b"producer", "status", b"status" ], ) -> None: ... diff --git a/temporalio/bridge/sdk-core b/temporalio/bridge/sdk-core index b2987e6a50..a7cbcbf363 160000 --- a/temporalio/bridge/sdk-core +++ b/temporalio/bridge/sdk-core @@ -1 +1 @@ -Subproject commit b2987e6a50f01bb373a1da90b3d490cad7b1bfae +Subproject commit a7cbcbf363697175b2cbb11ddae9961cd2f35284 diff --git a/temporalio/streams/_close.py b/temporalio/streams/_close.py index 2a9ecaecb9..c1face3436 100644 --- a/temporalio/streams/_close.py +++ b/temporalio/streams/_close.py @@ -10,8 +10,8 @@ class StreamCloseStatus(IntEnum): """Why a Stream closed, on a :py:class:`temporalio.streams.Closed` item. - The memory and Redis backends give ``CLOSED``. The owner statuses and a close failure come - from the native runtime. See :py:class:`temporalio.api.enums.v1.StreamCloseStatus`. + The memory and Redis backends give ``CLOSED``. Native Streams add their own statuses later. + See :py:class:`temporalio.api.enums.v1.StreamCloseStatus`. .. warning:: This API is experimental. @@ -21,19 +21,3 @@ class StreamCloseStatus(IntEnum): temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_UNSPECIFIED ) CLOSED = int(temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_CLOSED) - OWNER_COMPLETED = int( - temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_COMPLETED - ) - OWNER_FAILED = int( - temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_FAILED - ) - OWNER_CANCELED = int( - temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_CANCELED - ) - OWNER_TERMINATED = int( - temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_TERMINATED - ) - OWNER_TIMED_OUT = int( - temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_OWNER_TIMED_OUT - ) - IDLE = int(temporalio.api.enums.v1.StreamCloseStatus.STREAM_CLOSE_STATUS_IDLE) diff --git a/temporalio/streams/_items.py b/temporalio/streams/_items.py index 7ef1b86a6c..9c465e7ef4 100644 --- a/temporalio/streams/_items.py +++ b/temporalio/streams/_items.py @@ -62,8 +62,7 @@ class Closed: """The close of a Stream, the last item of a read. A read of a closed Stream ends after this item. On the memory and Redis backends the status is - :py:attr:`StreamCloseStatus.StreamCloseStatusClosed` and ``failure`` is ``None``. The owner - statuses and a close failure come from the native runtime. + :py:attr:`StreamCloseStatus.CLOSED`. Native Streams add their own statuses later. .. warning:: This API is experimental. @@ -77,6 +76,3 @@ class Closed: status: StreamCloseStatus """Why the Stream closed.""" - - failure: BaseException | None = None - """The owner's failure for ``OWNER_FAILED``, or ``None``.""" diff --git a/temporalio/streams/_reader.py b/temporalio/streams/_reader.py index 213fd41873..ae05c7447d 100644 --- a/temporalio/streams/_reader.py +++ b/temporalio/streams/_reader.py @@ -273,5 +273,4 @@ async def _item( status=StreamCloseStatus(int(close.status)) if close.status is not None else StreamCloseStatus.UNSPECIFIED, - failure=close.failure, ) diff --git a/temporalio/streams/system/models.py b/temporalio/streams/system/models.py index 6e5cac8e08..809076fd34 100644 --- a/temporalio/streams/system/models.py +++ b/temporalio/streams/system/models.py @@ -32,12 +32,6 @@ class StreamReadStart(enum.IntEnum): class StreamCloseStatus(enum.IntEnum): StreamCloseStatusUnspecified = 0 StreamCloseStatusClosed = 1 - StreamCloseStatusOwnerCompleted = 2 - StreamCloseStatusOwnerFailed = 3 - StreamCloseStatusOwnerCanceled = 4 - StreamCloseStatusOwnerTerminated = 5 - StreamCloseStatusOwnerTimedOut = 6 - StreamCloseStatusIdle = 7 def _stream_close_status_from_proto(value: int) -> StreamCloseStatus: @@ -1426,8 +1420,6 @@ class StreamClose: """The close of a Stream. A read gives it last.""" producer: Producer | None = None - result: typing.Any | None = None - failure: BaseException | None = None status: StreamCloseStatus | None = None @@ -1452,12 +1444,6 @@ def from_transfer_type( ) if value.HasField("producer") else None, - result=_support.payload_from_proto(value.result) - if value.HasField("result") - else None, - failure=_support.failure_from_proto(value.failure) - if value.HasField("failure") - else None, status=_stream_close_status_from_proto(value.status) if value.status != 0 else None, @@ -1473,10 +1459,6 @@ def to_transfer_type( message.producer.CopyFrom( _ProducerTransferTypeConverter().to_transfer_type(value.producer) ) - if value.result is not None: - message.result.CopyFrom(_support.payload_to_proto(value.result)) - if value.failure is not None: - message.failure.CopyFrom(_support.failure_to_proto(value.failure)) if value.status is not None: message.status = ( temporalio.api.enums.v1.stream_pb2.StreamCloseStatus.ValueType( diff --git a/tests/contrib/streams/test_refs.py b/tests/contrib/streams/test_refs.py index 6a562cb548..3000a05efa 100644 --- a/tests/contrib/streams/test_refs.py +++ b/tests/contrib/streams/test_refs.py @@ -102,8 +102,7 @@ def test_read_items_have_a_kind_that_narrows_their_type(): ) closed = streams.Closed( offset=streams.StreamOffset.parse("4"), - status=streams.StreamCloseStatus.OWNER_COMPLETED, - failure=None, + status=streams.StreamCloseStatus.CLOSED, ) items: list[streams.Record[Turn] | streams.Closed] = [record, closed] shown: list[Turn] = [] @@ -115,7 +114,7 @@ def test_read_items_have_a_kind_that_narrows_their_type(): case _: shown.append(item.data) assert shown == [Turn("model")] - assert ended == [streams.StreamCloseStatus.OWNER_COMPLETED] + assert ended == [streams.StreamCloseStatus.CLOSED] assert record.kind is streams.ItemKind.RECORD assert closed.kind is streams.ItemKind.CLOSED From b385be1406cd5e0e60577287f1a2bb17990d5a7f Mon Sep 17 00:00:00 2001 From: Moe Dashti Date: Sun, 8 Nov 2026 22:44:53 -0700 Subject: [PATCH 7/7] Pinned `temporalio-streams-redis` at the pushed removal rev. --- temporalio/bridge/Cargo.lock | 2 +- temporalio/bridge/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/temporalio/bridge/Cargo.lock b/temporalio/bridge/Cargo.lock index 464c895cb2..c6b9aa84e6 100644 --- a/temporalio/bridge/Cargo.lock +++ b/temporalio/bridge/Cargo.lock @@ -2954,7 +2954,7 @@ dependencies = [ [[package]] name = "temporalio-streams-redis" version = "0.1.0" -source = "git+https://github.com/temporal-community/sdk-streams?rev=dc93cba6bd49c89bb255a3aae9f4d970f4ffc997#dc93cba6bd49c89bb255a3aae9f4d970f4ffc997" +source = "git+https://github.com/temporal-community/sdk-streams?rev=03b93fae17f527986a9ffa8824927da4816b2fba#03b93fae17f527986a9ffa8824927da4816b2fba" dependencies = [ "async-trait", "aws-lc-rs", diff --git a/temporalio/bridge/Cargo.toml b/temporalio/bridge/Cargo.toml index a2aa91c3f7..7117aed91e 100644 --- a/temporalio/bridge/Cargo.toml +++ b/temporalio/bridge/Cargo.toml @@ -98,7 +98,7 @@ optional = true # The Redis Store, which lives in its own repo. [dependencies.temporalio-streams-redis] git = "https://github.com/temporal-community/sdk-streams" -rev = "dc93cba6bd49c89bb255a3aae9f4d970f4ffc997" +rev = "03b93fae17f527986a9ffa8824927da4816b2fba" default-features = false optional = true