diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..f0b0d90ac --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,62 @@ +name: docs + +# build sphinx docs on every PR + push to main; +# deploy to gh-pages only from main pushes. +# (see goodboy/tractor#123 for the original ask) +on: + push: + branches: + - main + pull_request: + # to run workflow manually from the "Actions" tab + workflow_dispatch: + +# needed by actions/deploy-pages +permissions: + contents: read + pages: write + id-token: write + +# never run >1 pages deploy at once +concurrency: + group: 'pages' + cancel-in-progress: true + +jobs: + build: + name: 'sphinx build' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install latest uv + uses: astral-sh/setup-uv@v6 + + # NOTE, no `d2` bin is installed in CI (yet) so + # the pre-rendered + committed SVGs under + # `docs/_diagrams/` are used as-is; see + # `docs/_ext/d2diagrams.py` for the fallback + # policy. + - name: Build html docs + run: | + uv sync --no-dev --group docs + uv run --no-dev --group docs make -C docs html + + - name: Upload pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: docs/_build/html + + deploy: + name: 'deploy to gh-pages' + if: ${{ github.ref == 'refs/heads/main' && github.event_name == 'push' }} + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy + id: deployment + uses: actions/deploy-pages@v4 diff --git a/ai/prompt-io/claude/20260611T175152Z_8526985c_prompt_io.md b/ai/prompt-io/claude/20260611T175152Z_8526985c_prompt_io.md new file mode 100644 index 000000000..3dfd43c40 --- /dev/null +++ b/ai/prompt-io/claude/20260611T175152Z_8526985c_prompt_io.md @@ -0,0 +1,93 @@ +--- +model: claude-fable-5[1m] +service: claude +session: 0780a862-e19a-4f0a-86cd-c8afc0997757 +timestamp: 2026-06-11T17:51:52Z +git_ref: 8526985c +scope: docs+code+config +substantive: true +raw_file: 20260611T175152Z_8526985c_prompt_io.raw.md +--- + +## Prompt + +> we need big boi docs; ours are way out of date and generally +> terrible. i'd like to use a sphinx theme in the vein and/or +> one-of/nearly-the-same-as used in one or all of the `msgspec`, +> `numpy`, `ray`, `xonsh`, `polars` projects. +> +> see the following oustanding but now very old issues to guide +> you: #175, #126, #123, #157 +> +> we should try to target a 3 columned sphinx theme with, +> - document index-nav on left, +> - body content in middle, +> - diagrams (ideally in d2lang if possible) on RHS +> +> optimize for (if possible), +> - reusing examples code without duplication in docs, +> - generating new examples from todos throughout code base, +> - distilling the essence of SC across processes in the simplest +> and most friendly way possible. +> +> do this work in a /open-wkt and do you best without my guidance +> for the first major pass - we will refine asap once you are +> complete. + +Session settings: `/effort max` + ultracode (multi-agent workflow +orchestration). Fully autonomous first pass; two orchestrated +agent fleets were used (6-agent recon survey, then 10-agent +content fan-out: 9 section writers + 1 examples smith), with the +orchestrator authoring the landing page, SC essay, sphinx +scaffold, d2 pipeline + diagram sources, CI workflow and all +integration/fix passes directly. + +## Response summary + +Complete sphinx docs revamp on branch `wkt/big_boi_docs`: +`pydata-sphinx-theme` 0.18 (3-column: left nav / content / +page-toc, with d2 diagrams + asides floated into the RHS margin +via custom CSS), sphinx 9.1, a local `.. d2::` directive +rendering `docs/diagrams/*.d2` sources with committed-SVG +fallback, a vendored `.. margin::` directive, ~25 new doc pages +(landing, start/, explain/, 12 guides, 10 api-ref pages, +project/), 5 new auto-tested examples + 3 modernized + 1 +renamed, and a gh-pages deploy workflow (issue #123). All +example code is `literalinclude`d from `examples/` (zero +duplication, CI-verified). Build: green, 24 warnings all +pre-existing-docstring/NEWS sourced. + +## Files changed + +See the branch diff (uncommitted at entry-write time): + +> `git diff test_cpu_throttling..wkt/big_boi_docs` +> `git -C status --short` (pre-commit working tree) + +- `docs/conf.py` — full rewrite for pydata theme + ext stack +- `docs/_ext/d2diagrams.py` — new `.. d2::` sphinx directive +- `docs/_ext/marginalia.py` — new `.. margin::` directive +- `docs/_static/css/custom.css` — b&w skin + RHS margin floats +- `docs/diagrams/*.d2` (7) — diagram sources (sketch/grayscale) +- `docs/_diagrams/*.svg` (7) — committed rendered fallbacks +- `docs/index.rst` — new landing (replaces dead-API doc) +- `docs/start/*.rst` (3), `docs/explain/*.rst` (3), + `docs/guide/*.rst` (13), `docs/api/*.rst` (10), + `docs/project/*.rst` (3) — new content tree +- `docs/dev_tips.rst` — removed (ported to project/dev-tips) +- `examples/{typed_payloads,nested_actor_tree, + service_daemon_discovery,uds_transport_actor_tree, + streaming_broadcast_fanout}.py` — new, smoke-tested +- `examples/{a_trynamic_first_scene, + actor_spawning_and_causality,parallelism/single_func}.py` — + `.result()` -> `.wait_for_result()` modernization +- `examples/parallelism/concurrent_futures_primes.py` — renamed + from leading-underscore + trio-runner shim added +- `pyproject.toml` — `docs` dependency-group filled in +- `uv.lock` — relock for docs group +- `.github/workflows/docs.yml` — build + gh-pages deploy + +## Human edits + +None yet — entry written pre-commit; the author reviews, stages +and commits manually (per repo workflow policy). diff --git a/ai/prompt-io/claude/20260611T175152Z_8526985c_prompt_io.raw.md b/ai/prompt-io/claude/20260611T175152Z_8526985c_prompt_io.raw.md new file mode 100644 index 000000000..90408f21c --- /dev/null +++ b/ai/prompt-io/claude/20260611T175152Z_8526985c_prompt_io.raw.md @@ -0,0 +1,49 @@ +--- +model: claude-fable-5[1m] +service: claude +timestamp: 2026-06-11T17:51:52Z +git_ref: 8526985c +diff_cmd: git diff test_cpu_throttling..wkt/big_boi_docs +--- + +# Raw output pointers (diff-ref mode) + +All generated content is code/config/docs committed alongside +this entry on branch `wkt/big_boi_docs`; per the prompt-io +diff-ref decision rule each file's verbatim content is the diff +itself rather than a copy here: + +> `git diff test_cpu_throttling..wkt/big_boi_docs -- docs/` +> `git diff test_cpu_throttling..wkt/big_boi_docs -- examples/` +> `git diff test_cpu_throttling..wkt/big_boi_docs -- pyproject.toml uv.lock .github/` + +## Generation notes (non-code output summary) + +- Theme research (web, agent-verified 2026-06-11): msgspec=furo, + xonsh=furo, numpy/ray/polars=pydata-sphinx-theme (ray migrated + off sphinx-book-theme); sphinx-book-theme 1.2.0 hard-pins + pydata 0.16.1 (stale) -> chose pydata 0.18 + sphinx 9.1. +- d2 ecosystem: no production-grade pypi extension exists + (sphinxcontrib-d2lang 0.0.5 ignores returncodes, uuid4 output + names; sphinx-d2 is an empty stub) -> wrote local + `docs/_ext/d2diagrams.py` (~230 LOC) with D2_BIN env + discovery, mtime caching, committed-SVG fallback and + literal-block last resort. +- Diagrams authored in d2 (theme-id 1 "Neutral Grey" + sketch + mode + ELK layout, validated by render + headless-firefox + screenshot loop): actor_tree, context_handshake (real + msg-spec names Start/StartAck/Started/Yield/Stop/Return), + streaming_pipeline, runtime_stack, debug_lock, + error_propagation, infected_aio. +- API truth enforced from a 6-agent recon pass over the + reorganized package tree (runtime/, discovery/, spawn/, ipc/, + msg/, devx/, trionics/): docs teach `.wait_for_result()`, + registrar (not arbiter) naming, `@tractor.context` + + `open_context()` as the core model, `run_in_actor()` as + convenience only. +- All ~30 literalincluded example scripts verified present; 9 + touched/new example files smoke-run green (exit 0, <16s). +- Final build: `sphinx-build -b html` succeeded; 24 residual + warnings, every one sourced from pre-existing library + docstring rst-isms or legacy NEWS.rst content (left untouched + by design; flagged for a follow-up docstring lint pass). diff --git a/docs/_diagrams/actor_tree.svg b/docs/_diagrams/actor_tree.svg new file mode 100644 index 000000000..bc09dc38a --- /dev/null +++ b/docs/_diagrams/actor_tree.svg @@ -0,0 +1,125 @@ + + + + + + + + + + + + +root actorsubactor'worker_0'subactor'worker_1'subactor'deeper'main()ActorNurserytrio task treeserve()ActorNurserytrio task tree opens opens spawns +supervises spawns +supervises spawns +supervises + + + + + + + diff --git a/docs/_diagrams/context_handshake.svg b/docs/_diagrams/context_handshake.svg new file mode 100644 index 000000000..04c7e9888 --- /dev/null +++ b/docs/_diagrams/context_handshake.svg @@ -0,0 +1,120 @@ + + + + + + + + + + + + +parent task(Portal)child actor task(@tractor.context fn) Start: open_context(fn, **kwargs) StartAck Started: await ctx.started(value) Yield: stream.send() Yield: stream.send() Stop: stream closed Return: fn return value + + + + + + + + + diff --git a/docs/_diagrams/debug_lock.svg b/docs/_diagrams/debug_lock.svg new file mode 100644 index 000000000..21a7e67eb --- /dev/null +++ b/docs/_diagrams/debug_lock.svg @@ -0,0 +1,118 @@ + + + + + + + + + + + + +subactor 'mary'root actor(owns the tty)subactor 'bob' tractor.pause(): acquire tty lock granted: pdb REPL is yours tractor.pause(): acquire tty lock continue: release lock granted: pdb REPL is yours + + + + + + + diff --git a/docs/_diagrams/error_propagation.svg b/docs/_diagrams/error_propagation.svg new file mode 100644 index 000000000..6684a4c39 --- /dev/null +++ b/docs/_diagrams/error_propagation.svg @@ -0,0 +1,117 @@ + + + + + + + + + + + + +root actor(ActorNursery)subactor 'gertie'(healthy, gets cancelled)subactor 'bobbie'raises NameError supervises supervises RemoteActorError(boxed NameError) cancel() + + + + + + diff --git a/docs/_diagrams/infected_aio.svg b/docs/_diagrams/infected_aio.svg new file mode 100644 index 000000000..66dc8b842 --- /dev/null +++ b/docs/_diagrams/infected_aio.svg @@ -0,0 +1,122 @@ + + + + + + + + + + + + +subactor process (infect_asyncio=True)parent actor (pure trio)asyncio event loopasyncio.Taskaio_echo_server()trio (guest mode)trio task@tractor.context fn LinkedTaskChannel.send()/.receive() IPC: Context + MsgStream + + + + diff --git a/docs/_diagrams/runtime_stack.svg b/docs/_diagrams/runtime_stack.svg new file mode 100644 index 000000000..e8da9bc24 --- /dev/null +++ b/docs/_diagrams/runtime_stack.svg @@ -0,0 +1,107 @@ + + + + + + + + + + + + +your app: plain trio tasks + nurseriestractor runtime: actors, portals,contexts + streams, RPCIPC Channel: msgspec-typed msgsover TCP | UDS transportsOS: one process per actor + + + diff --git a/docs/_diagrams/streaming_pipeline.svg b/docs/_diagrams/streaming_pipeline.svg new file mode 100644 index 000000000..a00159472 --- /dev/null +++ b/docs/_diagrams/streaming_pipeline.svg @@ -0,0 +1,123 @@ + + + + + + + + + + + + +actor 'streamer_0'actor 'streamer_1'actor 'aggregator'root actorstream_data(0)stream_data(1)aggregate()main() async for:yielded ints async for:yielded ints dedupedstream + + + + + diff --git a/docs/_ext/d2diagrams.py b/docs/_ext/d2diagrams.py new file mode 100644 index 000000000..02e4d7d59 --- /dev/null +++ b/docs/_ext/d2diagrams.py @@ -0,0 +1,235 @@ +# tractor: distributed structured concurrency. +''' +``.. d2::`` - embed d2lang_ diagrams in sphinx docs +with build-time rendering and a committed-SVG +fallback. + +Rendering policy, + +- when a ``d2`` binary is found (see discovery order + below) any out-of-date SVG is (re)rendered from its + ``.d2`` source (normally kept in ``docs/diagrams/``) + into ``docs/_diagrams/``, +- otherwise any pre-rendered (and git committed) SVG + already in ``docs/_diagrams/`` is used as-is, +- when neither is possible the diagram *source* is + emitted as a literal block so no content is ever + silently dropped. + +Binary discovery order, + +- the ``D2_BIN`` env var; may contain args which are + split via `shlex`, eg. + ``D2_BIN='nix run nixpkgs#d2 --'``, +- the ``d2_bin`` sphinx config value, +- ``shutil.which('d2')``. + +Usage, + +.. code:: rst + + .. d2:: diagrams/actor_tree.d2 + :caption: A tree of ``trio``-task-trees. + :margin: + +.. _d2lang: https://d2lang.com + +''' +from __future__ import annotations + +import os +from pathlib import Path +import shlex +import shutil +import subprocess as sp + +from docutils import nodes +from docutils.parsers.rst import directives +from sphinx.application import Sphinx +from sphinx.util import logging +from sphinx.util.docutils import SphinxDirective + +log = logging.getLogger(__name__) + +# subdir (under the sphinx srcdir) holding rendered, +# git-committed, fallback SVG outputs. +_outdir: str = '_diagrams' + + +def find_d2( + app: Sphinx, +) -> list[str]|None: + ''' + Resolve the d2 render command as an argv list or + `None` when no binary can be found. + + ''' + if env_bin := os.environ.get('D2_BIN'): + return shlex.split(env_bin) + if cfg_bin := app.config.d2_bin: + return shlex.split(cfg_bin) + if path_bin := shutil.which('d2'): + return [path_bin] + return None + + +def render_svg( + app: Sphinx, + src: Path, + out: Path, +) -> bool: + ''' + Maybe (re)render `src` -> `out`, returning + `True` when an up-to-date SVG exists after the + call. + + ''' + stale: bool = ( + not out.exists() + or + src.stat().st_mtime > out.stat().st_mtime + ) + if not stale: + return True + d2cmd: list[str]|None = find_d2(app) + if d2cmd is None: + if out.exists(): + log.info( + f'no d2 binary; using committed svg ' + f'for {src.name}' + ) + return True + return False + out.parent.mkdir( + parents=True, + exist_ok=True, + ) + argv: list[str] = ( + d2cmd + + list(app.config.d2_args) + + [str(src), str(out)] + ) + try: + proc = sp.run( + argv, + capture_output=True, + text=True, + timeout=120, + ) + except ( + OSError, + sp.TimeoutExpired, + ) as err: + log.warning( + f'd2 invocation failed for {src.name}: ' + f'{err}' + ) + return out.exists() + if proc.returncode != 0: + log.warning( + f'd2 render error for {src.name}:\n' + f'{proc.stderr}' + ) + return out.exists() + return True + + +class D2Diagram(SphinxDirective): + ''' + Render a ``.d2`` source file (path relative to + the sphinx srcdir) as an SVG figure. + + ''' + required_arguments = 1 + has_content = False + option_spec = { + 'caption': directives.unchanged, + 'alt': directives.unchanged, + 'width': ( + directives.length_or_percentage_or_unitless + ), + 'margin': directives.flag, + 'class': directives.class_option, + 'name': directives.unchanged, + } + + def run(self) -> list[nodes.Node]: + relsrc: str = self.arguments[0] + srcdir = Path(self.env.srcdir) + src: Path = srcdir / relsrc + self.env.note_dependency(relsrc) + if not src.exists(): + err = self.state_machine.reporter.error( + f'd2 source not found: {relsrc}', + line=self.lineno, + ) + return [err] + out: Path = ( + srcdir + / _outdir + / f'{src.stem}.svg' + ) + if not render_svg(self.env.app, src, out): + # last resort: emit the raw d2 source. + log.warning( + f'no svg available for {relsrc}; ' + f'emitting d2 source as literal block' + ) + literal = nodes.literal_block( + src.read_text(), + src.read_text(), + ) + literal['language'] = 'text' + return [literal] + img = nodes.image( + uri=f'/{_outdir}/{out.name}', + alt=self.options.get( + 'alt', + f'd2 diagram: {src.stem}', + ), + ) + if width := self.options.get('width'): + img['width'] = width + fig = nodes.figure() + fig += img + classes: list[str] = ( + ['d2-diagram'] + + self.options.get('class', []) + ) + if 'margin' in self.options: + # NB: the bare 'margin' class is what + # book-style themes key off for + # right-margin placement; our custom css + # uses 'd2-margin'. + classes += [ + 'margin', + 'd2-margin', + ] + fig['classes'] += classes + if caption_txt := self.options.get('caption'): + ( + inline_nodes, + _msgs, + ) = self.state.inline_text( + caption_txt, + self.lineno, + ) + caption = nodes.caption( + caption_txt, + '', + *inline_nodes, + ) + fig += caption + self.add_name(fig) + return [fig] + + +def setup(app: Sphinx) -> dict: + app.add_config_value('d2_bin', None, 'env') + app.add_config_value('d2_args', [], 'env') + app.add_directive('d2', D2Diagram) + return { + 'version': '0.1.0', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/_ext/marginalia.py b/docs/_ext/marginalia.py new file mode 100644 index 000000000..987f7264c --- /dev/null +++ b/docs/_ext/marginalia.py @@ -0,0 +1,51 @@ +# tractor: distributed structured concurrency. +''' +``.. margin::`` - prose-anchored, right-margin asides. + +A theme-agnostic vendoring of `sphinx_book_theme`'s +`Margin` directive: a `docutils` `Sidebar` subclass +which tags the node with a ``margin`` class; placement +is then pure CSS (see ``_static/css/custom.css``) +allowing use on any theme incl. our +`pydata_sphinx_theme`. + +Usage, + +.. code:: rst + + .. margin:: An optional title + + Aside content; text, figures, whatever. + +''' +from docutils import nodes +from docutils.parsers.rst.directives.body import ( + Sidebar, +) +from sphinx.application import Sphinx + + +class Margin(Sidebar): + ''' + Notes/figures placed in the right margin, anchored + at the current point in the prose flow. + + ''' + required_arguments = 0 + optional_arguments = 1 + + def run(self) -> list[nodes.Node]: + if not self.arguments: + self.arguments = [''] + out: list[nodes.Node] = super().run() + out[0].attributes['classes'].append('margin') + return out + + +def setup(app: Sphinx) -> dict: + app.add_directive('margin', Margin) + return { + 'version': '0.1.0', + 'parallel_read_safe': True, + 'parallel_write_safe': True, + } diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css new file mode 100644 index 000000000..b75c49f78 --- /dev/null +++ b/docs/_static/css/custom.css @@ -0,0 +1,75 @@ +/* tractor docs: a minimal black + white skin over + * `pydata-sphinx-theme` plus RHS-marginalia + d2 + * diagram styling. + */ +html[data-theme="light"] { + --pst-color-primary: #000000; + --pst-color-secondary: #3d3d3d; + --pst-color-accent: #5a5a5a; + --pst-color-link: #000000; + --pst-color-link-hover: #5a5a5a; + --pst-color-inline-code: #1a1a1a; + --pst-color-inline-code-links: #000000; +} +html[data-theme="dark"] { + --pst-color-primary: #ffffff; + --pst-color-secondary: #c9c9c9; + --pst-color-accent: #a8a8a8; + --pst-color-link: #ffffff; + --pst-color-link-hover: #bdbdbd; + --pst-color-inline-code: #e8e8e8; + --pst-color-inline-code-links: #ffffff; +} +/* mono-chrome links: rely on underline for + * affordance instead of color. + */ +.bd-content a:not(.headerlink) { + text-decoration: underline; + text-underline-offset: 0.18em; +} +/* d2 diagram figures */ +figure.d2-diagram { + text-align: center; +} +figure.d2-diagram img { + max-width: 100%; + height: auto; +} +/* keep light-rendered svgs legible in dark mode */ +html[data-theme="dark"] figure.d2-diagram img { + background: #ffffff; + border-radius: 6px; + padding: 6px; +} +/* prose-anchored right-margin asides (tufte-ish): + * float right within the content column on wide + * screens, collapse inline on narrow ones. + */ +@media (min-width: 960px) { + aside.margin, + figure.margin, + div.margin, + figure.d2-margin { + float: right; + clear: right; + width: 44%; + margin: 0.2rem 0 1rem 1.4rem; + font-size: 0.85rem; + } +} +/* strip docutils sidebar chrome from margin asides */ +aside.sidebar.margin { + border: none; + background: transparent; + padding: 0; +} +aside.sidebar.margin > p.sidebar-title { + font-weight: 600; + font-size: 0.9rem; + margin-bottom: 0.3rem; +} +/* landing page hero logo sizing */ +img.hero-logo { + max-width: 360px; + width: 60%; +} diff --git a/docs/api/context.rst b/docs/api/context.rst new file mode 100644 index 000000000..ff024bd97 --- /dev/null +++ b/docs/api/context.rst @@ -0,0 +1,134 @@ +Contexts and streaming +====================== + +The modern core of ``tractor``: a :class:`Context` links a task in +one actor to a task in another as a *single* structured concurrency +(SC) scope stretched across the IPC boundary — errors, results and +cancellation flow between the pair `exactly like trio`_ tasks under +a common `nursery`_. Open one with ``Portal.open_context()`` (see +:func:`tractor._context.open_context_from_portal` in +:doc:`/api/core`), then optionally bridge a bidirectional +:class:`MsgStream` between the two tasks. + +.. d2:: diagrams/context_handshake.d2 + :caption: The ``open_context()`` <-> ``ctx.started()`` handshake. + :margin: + :alt: parent and child actor context handshake sequence + +For the guided, example-driven tour see :doc:`/guide/context`; this +page is the precise API surface. + +.. currentmodule:: tractor + +The ``@context`` decorator +-------------------------- + +.. autofunction:: context + +.. note:: + + The decorated function **must** declare a parameter annotated + ``tractor.Context`` (any param name works); the runtime injects + the context instance there on each remote invocation. Pass + ``pld_spec`` to type-restrict (and validate) the payloads this + endpoint may shuttle — violations raise + :class:`MsgTypeError`. See ``examples/typed_payloads.py``. + +``Context`` +----------- + +.. autoclass:: Context + :members: started, + wait_for_result, + cancel, + cid, + chan, + side, + cancel_called, + cancelled_caught, + cancel_acked, + canceller, + maybe_error, + outcome + +.. deprecated:: 0.1.0a6 + + ``Context.result()`` warns; use :meth:`Context.wait_for_result`. + +.. note:: + + A :class:`Context` is **not** a :class:`trio.CancelScope`: + :meth:`Context.cancel` requests cancellation of the *remote* + peer task and does not cancel the local scope. If *you* + requested the cancel, the resulting :class:`ContextCancelled` + is absorbed at ``open_context()`` exit; a cancel originating + anywhere else (the peer, or a third-party actor recorded in + :attr:`ContextCancelled.canceller`) *is* raised locally. This + self-vs-cross-cancel rule is the key to writing correct + inter-actor teardown logic — see :doc:`/guide/context`. + +Bidirectional streaming +----------------------- + +.. autofunction:: tractor._streaming.open_stream_from_ctx + +.. note:: + + :func:`~tractor._streaming.open_stream_from_ctx` is bound as + the **method-alias** ``Context.open_stream()`` — call it as + ``async with ctx.open_stream() as stream:``. Both sides of the + context must enter it for the dialog to be open. + +.. autoclass:: MsgStream + :members: send, + receive, + receive_nowait, + aclose, + subscribe, + ctx, + closed + +.. note:: + + A :class:`MsgStream` is one-shot use: once closed it can never + be "re-opened" — open a fresh :class:`Context` instead. Remote + end-of-stream surfaces as :class:`StopAsyncIteration` from + ``async for``; un-consumed sends overrun the receiver and raise + :class:`tractor._exceptions.StreamOverrun` unless the context + was opened with ``allow_overruns=True``. + +:meth:`MsgStream.subscribe` fans a single IPC stream out to +multiple *local* tasks via a +:class:`tractor.trionics.BroadcastReceiver` (see +:doc:`/api/trionics`); the underlying allocation is idempotent and +non-reversible for the stream's lifetime. See +``examples/streaming_broadcast_fanout.py`` for the pattern in +action. + +Legacy one-way streaming +------------------------ + +.. autofunction:: stream + +.. warning:: + + ``@tractor.stream`` and ``Portal.open_stream_from()`` are the + *legacy* one-way streaming API kept for backward compat: a + plain async-generator function streamed parent-ward with no + child-side receive leg. New code should use + ``@tractor.context`` + ``ctx.open_stream()`` (bidirectional, + SC-linked, typed). Note ``ctx`` is now a reserved param name + for ``@context`` endpoints — ``@stream`` functions must use + ``stream`` instead, and ``ctx.send_yield()`` is deprecated in + favor of :meth:`MsgStream.send`. + +.. seealso:: + + :doc:`/api/errors` for :class:`ContextCancelled` / + :class:`MsgTypeError` semantics, :doc:`/api/msg` for payload + typing via ``pld_spec`` and codecs, :doc:`/api/trionics` for + the broadcast fan-out machinery, and the guided tours in + :doc:`/guide/streaming` + :doc:`/guide/cancellation`. + +.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics +.. _nursery: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning diff --git a/docs/api/core.rst b/docs/api/core.rst new file mode 100644 index 000000000..196da2d05 --- /dev/null +++ b/docs/api/core.rst @@ -0,0 +1,157 @@ +Runtime and spawning +==================== + +The core lifecycle API: boot the runtime in your root process, +spawn trio-"actors" (processes running ``trio.run()`` task trees) +under a one-cancels-all supervisor, and talk to them through +portals. This is structured concurrency (SC) applied *transitively*: +every spawned process is owned by a nursery block and errors +`always propagate`_. If you can create zombies it **is a bug**. + +.. currentmodule:: tractor + +Booting the runtime +------------------- + +.. autofunction:: open_root_actor + +.. note:: + + The env vars ``TRACTOR_LOGLEVEL`` and ``TRACTOR_SPAWN_METHOD`` + override the ``loglevel`` / ``start_method`` params so you can + crank verbosity or swap spawn backends without touching app + code. Exactly **one** IPC transport may be enabled per actor + (see ``enable_transports`` and :doc:`/api/ipc`). + +.. autofunction:: run_daemon + +Spawning actors +--------------- + +.. d2:: diagrams/actor_tree.d2 + :caption: A supervised actor (process) tree. + :margin: + :alt: root actor supervising a tree of subactors + +.. autofunction:: open_nursery + +.. autoclass:: ActorNursery + :members: start_actor, + run_in_actor, + cancel, + cancel_called, + cancelled_caught + +.. note:: + + :meth:`ActorNursery.start_actor` (daemon actor + portal) is the + blessed spawning primitive; pair it with + ``Portal.open_context()`` for SC-linked remote tasks. + :meth:`ActorNursery.run_in_actor` is a *convenience* one-shot — + spawn, run a single task, auto-cancel after the result — slated + to be rebuilt as a high-level wrapper, so don't design around + it as the core model. + +.. deprecated:: 0.1.0a6 + + ``ActorNursery.cancelled`` warns; use + :attr:`ActorNursery.cancel_called` and + :attr:`ActorNursery.cancelled_caught`. The ``rpc_module_paths`` + kwarg is likewise deprecated in favor of ``enable_modules``. + +Portals +------- + +A :class:`Portal` "opens a portal" into a peer actor's memory +domain: you call functions and start SC-linked tasks *over IPC* as +though they were local, with results, errors and cancellation +flowing back `exactly like trio`_. + +.. autoclass:: Portal + :members: run, + run_from_ns, + open_stream_from, + wait_for_result, + cancel_actor, + chan + +.. deprecated:: 0.1.0a6 + + ``Portal.result()`` warns; use :meth:`Portal.wait_for_result`. + The str-form ``Portal.run('mod.path', 'fn_name')`` also warns; + pass a function *object* whose module is listed in the target's + ``enable_modules``. ``Portal.channel`` is the legacy spelling + of :attr:`Portal.chan`. + +.. autofunction:: tractor._context.open_context_from_portal + +.. note:: + + :func:`~tractor._context.open_context_from_portal` is bound as + the **method-alias** ``Portal.open_context()`` — that's the + spelling you should actually call: + ``portal.open_context(fn, **kwargs)``. See :doc:`/api/context` + for the full ``Context`` + ``MsgStream`` API it unlocks. + +.. note:: + + :meth:`Portal.cancel_actor` cancels the *whole* remote runtime + and process (machine-level), not a single task — use + :meth:`Context.cancel` for task-level cancellation. Pass + ``raise_on_timeout=True`` to get an ``ActorTooSlowError`` you + can escalate per SC discipline (see :doc:`/api/errors`). + +Clusters +-------- + +.. autofunction:: open_actor_cluster + +Spawn a *flat* cluster of ``count`` worker actors (default: one +per core) all serving the RPC ``modules`` list, yielding a +``dict[str, Portal]`` keyed by actor name. Handy for +embarrassingly parallel fan-out; see ``examples/quick_cluster.py``. + +Runtime introspection +--------------------- + +.. autofunction:: current_actor + +.. autoclass:: Actor + :members: aid, + name, + uid, + is_registrar, + is_infected_aio, + cancel_soon + +.. note:: + + :class:`Actor` is the per-process runtime singleton (msg loop, + RPC scheduling, IPC server) — you never instantiate it yourself + and should normally only touch the identity/introspection + surface listed above. The canonical identity type is + :attr:`Actor.aid` (a ``tractor.msg.Aid`` struct); + :attr:`Actor.uid` is the legacy ``(name, uuid)`` 2-tuple which + is still pervasive in logs and error metadata. + +.. deprecated:: 0.1.0a6 + + ``Actor.is_arbiter`` warns; use :attr:`Actor.is_registrar`. + The ``arbiter_addr`` constructor kwarg is deprecated for + ``registry_addrs``. + +.. autofunction:: current_ipc_ctx + +.. autofunction:: is_root_process + +.. autofunction:: get_runtime_vars + +.. seealso:: + + :doc:`/api/context` for the SC-linked remote task API, + :doc:`/api/discovery` for finding actors by name, and the + guided tours in :doc:`/guide/spawning`, :doc:`/guide/rpc` and + :doc:`/guide/context`. + +.. _always propagate: https://trio.readthedocs.io/en/latest/design.html#exceptions-always-propagate +.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics diff --git a/docs/api/devx.rst b/docs/api/devx.rst new file mode 100644 index 000000000..b45bcb875 --- /dev/null +++ b/docs/api/devx.rst @@ -0,0 +1,92 @@ +Debugging and devx: ``tractor.devx`` +==================================== + +Multi-process debugging that actually works: boot the tree with +``open_root_actor(debug_mode=True)`` (or pass it to +``open_nursery()``) and any crash or explicit pause in *any* actor +acquires a tree-global TTY lock and drops you into a +`pdbp`_-powered REPL — one actor at a time, ``SIGINT`` shielded, +no garbled terminals. The top-level helpers below are the daily +drivers; the rest of the toolbox lives under ``tractor.devx``. + +.. currentmodule:: tractor + +Pausing and post-mortems +------------------------ + +.. autofunction:: pause + +.. autofunction:: pause_from_sync + +.. note:: + + :func:`pause_from_sync` needs the `greenback`_ portal: boot + with ``open_root_actor(maybe_enable_greenback=True)`` (mind + the `performance implications`_). With ``debug_mode`` on, the + built-in ``breakpoint()`` is also remapped to a + ``tractor``-safe equivalent. + +.. autofunction:: post_mortem + +.. deprecated:: 0.1.0a6 + + ``tractor.breakpoint()`` warns and simply calls :func:`pause` + — use :func:`tractor.pause` (async) or + :func:`tractor.pause_from_sync` in new code. + +Crash handling for CLIs and sync entrypoints +-------------------------------------------- + +.. currentmodule:: tractor.devx + +.. autofunction:: open_crash_handler + +.. autofunction:: maybe_open_crash_handler + +.. note:: + + Both are *sync* context managers usable before (or without) + ``trio.run()`` — wrap your CLI ``main()`` to get a post-mortem + REPL on any uncaught exception instead of a bare traceback. + +Runtime hang-hunting +-------------------- + +.. autofunction:: enable_stack_on_sig + +With stackscope_ integration enabled (also via +``open_root_actor(enable_stack_on_sig=True)`` or the +``TRACTOR_ENABLE_STACKSCOPE`` env var) a ``SIGUSR1`` triggers a +full trio task-tree dump from every actor — works on live, +*non*-debug-mode trees too: + +.. code:: sh + + pkill --signal SIGUSR1 -f + +Dumps also tee to ``/tmp/tractor-stackscope-.log`` so you +still get output under captured/CI stdio. + +Lower-level debug plumbing +-------------------------- + +.. autofunction:: mk_pdb + +.. autofunction:: maybe_wait_for_debugger + +:func:`maybe_wait_for_debugger` is mainly useful in runtime/test +code that must avoid tearing down a tree while a child still +holds the global debug lock. + +.. seealso:: + + :doc:`/api/errors` for the boxed error types you'll inspect + from the REPL, :doc:`/api/core` for the ``debug_mode`` / + ``maybe_enable_greenback`` / ``enable_stack_on_sig`` boot + flags on :func:`tractor.open_root_actor`, and + :doc:`/guide/debugging` for the guided multi-actor REPL tour. + +.. _pdbp: https://github.com/mdmintz/pdbp +.. _greenback: https://greenback.readthedocs.io/en/latest/ +.. _performance implications: https://greenback.readthedocs.io/en/latest/principle.html#performance +.. _stackscope: https://github.com/oremanj/stackscope diff --git a/docs/api/discovery.rst b/docs/api/discovery.rst new file mode 100644 index 000000000..b054c4d43 --- /dev/null +++ b/docs/api/discovery.rst @@ -0,0 +1,67 @@ +Discovery and the registrar +=========================== + +Every actor registers its ``(name, uuid)`` and transport addresses +with a *registrar* actor — by default the root of the tree, or +whichever actor serves at the ``registry_addrs`` you boot with. +The discovery API lets any actor look up any other **by name** and +get back a connected :class:`~tractor.Portal`, giving you +service-discovery patterns (daemons, service trees, multi-host +meshes) without hard-coding addresses. + +Lookups first scan already-connected peers before RPC-ing the +registrar, and multihomed results are ranked UDS > local TCP > +remote TCP. See ``examples/service_daemon_discovery.py`` for the +canonical daemon + lookup pattern. + +.. currentmodule:: tractor + +Lookup APIs +----------- + +.. autofunction:: find_actor + +.. autofunction:: wait_for_actor + +.. autofunction:: query_actor + +.. autofunction:: get_registry + +.. note:: + + :func:`find_actor` yields ``None`` when nothing is registered + under the name (or raises with ``raise_on_none=True``); + :func:`wait_for_actor` blocks until the name appears; + :func:`query_actor` only *looks up* the address without + connecting to the target. + +The ``Registrar`` +----------------- + +.. autoclass:: Registrar + :show-inheritance: + +A :class:`Registrar` is just an :class:`~tractor.Actor` subtype +maintaining the name -> addresses table; you rarely touch it +directly beyond passing ``registry_addrs`` / +``ensure_registry=True`` to :func:`~tractor.open_root_actor`. +Check :attr:`Actor.is_registrar ` to +ask "am I it?". + +Legacy ``Arbiter`` alias +------------------------ + +.. deprecated:: 0.1.0a6 + + ``tractor.Arbiter`` survives only as a class alias of + :class:`Registrar` and all "arbiter" terminology is replaced + by "registrar"/"registry" across the API: ``get_arbiter()`` is + removed (use :func:`get_registry`) and the ``arbiter_addr`` + kwarg is replaced by ``registry_addrs``. + +.. seealso:: + + :doc:`/api/core` for booting a registrar via + ``open_root_actor(registry_addrs=...)``, :doc:`/api/ipc` for + the transport/address model the registry stores, and + :doc:`/guide/discovery` for the worked walkthrough. diff --git a/docs/api/errors.rst b/docs/api/errors.rst new file mode 100644 index 000000000..abfb0966a --- /dev/null +++ b/docs/api/errors.rst @@ -0,0 +1,102 @@ +Errors and cancellation types +============================= + +``tractor`` extends trio's "exceptions `always propagate`_" rule +across the process boundary: a crash in any actor is serialized as +an ``Error`` msg, shuttled over IPC, and re-raised in the linked +parent scope as a *boxed* :class:`RemoteActorError` — preserving +the original type, traceback text and source-actor identity, even +across multi-hop relays (a.k.a. "inceptions"). + +The most-used types below are importable from ``tractor`` +directly; the remainder live in ``tractor._exceptions`` (not yet +re-exported at top level). + +.. currentmodule:: tractor + +Boxed remote errors +------------------- + +.. autoexception:: RemoteActorError + :members: boxed_type, + src_uid, + relay_uid, + pformat + +.. code:: python + + try: + async with portal.open_context(ep_fn) as (ctx, first): + ... + except tractor.RemoteActorError as rae: + if rae.boxed_type is ValueError: + ... # remote task raised a `ValueError` + +.. autoexception:: ContextCancelled + :show-inheritance: + :members: canceller + +.. note:: + + Inspect :attr:`ContextCancelled.canceller` (the requesting + actor's uid) to distinguish a *self*-requested cancel (absorbed + at ``open_context()`` exit) from a *cross*-actor cancel (raised + locally) — the full rules live in :doc:`/api/context`. + +Typed-messaging errors +---------------------- + +.. autoexception:: MsgTypeError + :show-inheritance: + :members: bad_msg, + expected_msg_type + +An "IPC ``TypeError``": a message failed validation against the +active msg-spec / ``pld_spec`` (see :doc:`/api/msg`). Raised +sender-side for control msgs (``Started``/``Return``) and +receiver-side for stream ``Yield`` payloads. + +.. autoexception:: tractor._exceptions.StreamOverrun + :show-inheritance: + +The sender out-paced the receiver's buffer on a +:class:`~tractor.MsgStream` opened without +``allow_overruns=True``; subtypes :class:`trio.TooSlowError`. + +Transport and runtime errors +---------------------------- + +.. autoexception:: TransportClosed + +.. autoexception:: ModuleNotExposed + :show-inheritance: + +Raised when an RPC requests a function from a module not listed +in the target actor's ``enable_modules`` allowlist — +capability-style access control, not an import bug on your end ;) + +.. autoexception:: tractor._exceptions.NoRuntime + :show-inheritance: + +Raised by :func:`tractor.current_actor` (and friends) when no +actor runtime is up in the current process. + +.. autoexception:: tractor._exceptions.ActorTooSlowError + :show-inheritance: + +A peer actor failed to ack a cancel request within the bounded +wait — the SC-sanctioned escalation signal from APIs like +``Portal.cancel_actor(raise_on_timeout=True)``. Catch it to +escalate (e.g. hard-kill via the supervising +:class:`~tractor.ActorNursery`); never just ignore it, that's how +zombies happen. + +.. seealso:: + + :doc:`/api/context` for how cancellation and errors flow + through a :class:`~tractor.Context`, :doc:`/api/devx` for + crash-handling REPL tooling (``debug_mode``, post-mortems), + and :doc:`/guide/cancellation` for the full SC-cancellation + story. + +.. _always propagate: https://trio.readthedocs.io/en/latest/design.html#exceptions-always-propagate diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 000000000..f2b90900c --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,67 @@ +API reference +============= + +This is the curated reference for ``tractor``'s public surface: the +names you can import and lean on without reading runtime internals. +Everything below is re-exported at the top level (``import +tractor``) unless a page says otherwise; subsystems like +``tractor.msg``, ``tractor.trionics``, ``tractor.to_asyncio``, +``tractor.devx`` and ``tractor.log`` are importable as submodules. + +``tractor`` is "just trio_" extended across processes: every API +here is designed to keep the structured concurrency (SC) rules you +already know from the `trio docs`_ intact across the process +boundary. If a name isn't documented here it's an internal — expect +it to change without notice B). + +.. currentmodule:: tractor + +Most-used names at a glance: + +.. autosummary:: + :nosignatures: + + open_root_actor + open_nursery + run_daemon + ActorNursery + Portal + context + Context + MsgStream + open_actor_cluster + find_actor + wait_for_actor + get_registry + current_actor + current_ipc_ctx + is_root_process + get_runtime_vars + RemoteActorError + ContextCancelled + MsgTypeError + pause + post_mortem + Channel + +.. toctree:: + :maxdepth: 1 + :caption: Reference pages + + core + context + discovery + errors + msg + trionics + to_asyncio + devx + ipc + +Where to next? If you're new, start with the runtime and spawning +APIs in :doc:`/api/core`, then graduate to the inter-actor task +linking model in :doc:`/api/context` — it's the heart of the whole +system. + +.. _trio: https://github.com/python-trio/trio +.. _trio docs: https://trio.readthedocs.io/en/latest/ diff --git a/docs/api/ipc.rst b/docs/api/ipc.rst new file mode 100644 index 000000000..a9a36f800 --- /dev/null +++ b/docs/api/ipc.rst @@ -0,0 +1,89 @@ +IPC and logging +=============== + +Under every portal, context and stream sits a per-peer +:class:`~tractor.Channel`: a msgpack-typed messaging link wrapping +one OS transport connection. Transports are pluggable per actor +via ``enable_transports=['tcp' | 'uds']`` — TCP is the default, +UDS (unix domain sockets) gives you port-less, same-host IPC with +kernel-provided peer credentials for free — and exactly **one** +transport may currently be enabled per actor. + +.. d2:: diagrams/runtime_stack.d2 + :caption: Where ``Channel`` sits in the runtime stack. + :margin: + :alt: layered runtime stack from app code down to transports + +Addresses are "unwrapped" tuples at the API edges: +``('host', port)`` for TCP, filesystem-path pairs for UDS. For +the full layering story — transport protocols, the IPC server, +address types and the msg loop — see +:doc:`/explain/architecture`. + +.. currentmodule:: tractor + +``Channel`` +----------- + +.. autoclass:: Channel + :members: from_addr, + send, + recv, + aclose, + connected, + apply_codec, + aid, + laddr, + raddr, + closed + +.. deprecated:: 0.1.0a6 + + ``Channel.uid`` warns; use :attr:`Channel.aid` which carries + richer (optional) identity fields beyond the legacy + ``(name, uuid)`` pair. + +.. note:: + + You rarely construct a :class:`Channel` yourself — the runtime + hands them out via :attr:`Portal.chan ` + and :attr:`Context.chan `. Treat the + send/recv surface as advanced API: normal apps should speak + :class:`~tractor.MsgStream` instead. + +Choosing a transport +-------------------- + +.. literalinclude:: ../../examples/uds_transport_actor_tree.py + :caption: examples/uds_transport_actor_tree.py + :language: python + +Logging +------- + +``tractor.log`` provides the structured, colorized console +logging used across the runtime — with actor-name + task-aware +record headers and extra log levels below :data:`logging.DEBUG` +(``'transport'``, ``'runtime'``, ``'cancel'``, ``'devx'``) for +spelunking the runtime itself. Use it for your app too: it's +already distributed-system aware. + +.. currentmodule:: tractor.log + +.. autofunction:: get_logger + +.. autofunction:: get_console_log + +.. note:: + + The ``TRACTOR_LOGLEVEL`` env var overrides any caller-passed + ``loglevel`` (e.g. to ``open_root_actor()``) so you can crank + console verbosity without touching code; subactors inherit + the root's level by default. + +.. seealso:: + + :doc:`/explain/architecture` for the transport/server + internals, :doc:`/api/discovery` for how channel addresses + get registered and found, and :doc:`/api/msg` for the codec + layer every channel speaks. diff --git a/docs/api/msg.rst b/docs/api/msg.rst new file mode 100644 index 000000000..4062b7020 --- /dev/null +++ b/docs/api/msg.rst @@ -0,0 +1,107 @@ +Typed messaging: ``tractor.msg`` +================================ + +All inter-actor communication rides a small, strictly-typed +msgpack wire protocol built from :class:`msgspec.Struct` types — +the "SC-shuttle" protocol that powers contexts, streams, RPC and +cancellation. You normally never touch these msg types directly +(the :class:`~tractor.Context` API speaks them for you) but you +*do* use this subpackage to define **payload type contracts**: per +endpoint via ``@tractor.context(pld_spec=...)`` or per channel via +custom codecs. + +Violations of an active msg-spec surface as +:class:`~tractor.MsgTypeError` (see :doc:`/api/errors`); the full +typed-payload workflow is shown in ``examples/typed_payloads.py``. + +.. currentmodule:: tractor.msg + +The protocol message set +------------------------ + +.. autosummary:: + :nosignatures: + + PayloadMsg + Aid + SpawnSpec + Start + StartAck + Started + Yield + Stop + Return + CancelAck + Error + +``Aid`` (identity handshake) and ``SpawnSpec`` (parent -> child +init) run at connection setup; ``Start``/``StartAck`` initiate an +RPC task; ``Started``/``Yield``/``Stop``/``Return`` are the +:class:`~tractor.Context` dialog phases; ``CancelAck`` and +``Error`` close the loop on cancellation and (boxed) failure. +``Msg`` is a legacy alias of ``PayloadMsg``. The union of all of +the above is exported as ``MsgType`` (also ``__msg_spec__``). + +.. automodule:: tractor.msg.types + :no-members: + +.. currentmodule:: tractor.msg + +Codec construction and override +------------------------------- + +.. autofunction:: mk_codec + +.. autoclass:: MsgCodec + :members: encode, + decode, + msg_spec + +.. autofunction:: mk_dec + +.. autoclass:: MsgDec + :members: decode, + spec + +.. autofunction:: apply_codec + +.. autofunction:: current_codec + +.. note:: + + :func:`apply_codec` swaps the codec via a + :class:`contextvars.ContextVar` — the override only applies to + the *current task* (and tasks it starts), not sibling tasks + already running in the actor. Payload-decoding is layered: the + outer codec leaves ``.pld`` fields as ``msgspec.Raw`` and each + context's payload-receiver decodes them against *its* spec + (the "cheap-or-nasty" validation pattern). + +Namespace pointers +------------------ + +.. autoclass:: NamespacePath + :members: from_ref, + load_ref, + to_tuple + +The ``'module.path:obj_name'`` :class:`str`-subtype used to +address every RPC target function over the wire (same format as +:func:`pkgutil.resolve_name`). + +Pretty structs +-------------- + +.. autoclass:: Struct + :show-inheritance: + +A :class:`msgspec.Struct` subtype with a multi-line pretty +``__repr__`` — handy as a base for your own IPC payload types so +crash logs stay readable. + +.. seealso:: + + :doc:`/api/context` for where ``pld_spec`` typing plugs into + the ``@context`` decorator, :doc:`/api/errors` for + :class:`~tractor.MsgTypeError` semantics, and + :doc:`/guide/msging` for the guided typed-messaging tour. diff --git a/docs/api/to_asyncio.rst b/docs/api/to_asyncio.rst new file mode 100644 index 000000000..05baad9ad --- /dev/null +++ b/docs/api/to_asyncio.rst @@ -0,0 +1,97 @@ +asyncio interop: ``tractor.to_asyncio`` +======================================= + +"Infected asyncio" mode: spawn an actor with +``start_actor(..., infect_asyncio=True)`` and its process runs +:mod:`trio` as a `guest`_ on top of the :mod:`asyncio` loop — +letting your trio task tree drive asyncio tasks *in the same +process* while the rest of the actor tree stays pure trio. Each +trio <-> asyncio task pair is linked with structured concurrency +(SC) semantics: error or cancellation on either side tears down +both, with the cause translated cross-loop. + +.. d2:: diagrams/infected_aio.d2 + :caption: A trio guest driving asyncio tasks in one actor. + :margin: + :alt: trio guest mode inside an asyncio-infected actor + +See ``examples/infected_asyncio_echo_server.py`` for a complete +worked example. + +.. currentmodule:: tractor.to_asyncio + +Starting asyncio tasks from trio +-------------------------------- + +.. autofunction:: open_channel_from + +.. autofunction:: run_task + +.. note:: + + :func:`open_channel_from` mirrors the + ``Portal.open_context()`` handshake: the asyncio side calls + ``chan.started_nowait(value)`` and that value pops out as + ``first`` on the trio side. :func:`run_task` is the one-shot + form — run a single asyncio-compatible coroutine fn and return + its result to trio. + +The inter-loop channel +---------------------- + +.. autoclass:: LinkedTaskChannel + :members: send, + receive, + wait_for_result, + subscribe, + started_nowait, + send_nowait, + get, + cancel_asyncio_task, + closed + +.. note:: + + The trio side uses the async API + (:meth:`LinkedTaskChannel.send` / + :meth:`LinkedTaskChannel.receive`); the asyncio side uses the + loop-safe sync/await mix + (:meth:`LinkedTaskChannel.send_nowait` / + :meth:`LinkedTaskChannel.get` / + :meth:`LinkedTaskChannel.started_nowait`). + +Translated exception types +-------------------------- + +Cross-loop failures are re-raised on the *other* side as one of +these explicit translation types, so you always know which loop +actually died first: + +.. autoexception:: tractor._exceptions.AsyncioCancelled + +.. autoexception:: tractor._exceptions.AsyncioTaskExited + +.. autoexception:: tractor._exceptions.TrioCancelled + +.. autoexception:: tractor._exceptions.TrioTaskExited + +.. autoexception:: AsyncioRuntimeTranslationError + :show-inheritance: + +Guest-mode entrypoint +--------------------- + +``run_as_asyncio_guest()`` is the runtime-internal entrypoint that +boots trio in `guest`_ mode inside an infected actor — you get it +implicitly via ``infect_asyncio=True`` and shouldn't need to call +it yourself. + +.. seealso:: + + :doc:`/api/core` for the ``infect_asyncio`` spawn flag, + :meth:`tractor.Actor.is_infected_aio` for runtime + introspection, :doc:`/api/devx` for using the debugger from + inside asyncio tasks, and :doc:`/guide/asyncio` for the + guided tour. + +.. _guest: https://trio.readthedocs.io/en/stable/reference-lowlevel.html?highlight=guest%20mode#using-guest-mode-to-run-trio-on-top-of-other-event-loops diff --git a/docs/api/trionics.rst b/docs/api/trionics.rst new file mode 100644 index 000000000..a92a290a3 --- /dev/null +++ b/docs/api/trionics.rst @@ -0,0 +1,76 @@ +Trio patterns: ``tractor.trionics`` +=================================== + +Sugary structured concurrency (SC) patterns for plain :mod:`trio` +code — **no actor runtime required**. These helpers grew out of +real distributed-system needs in ``tractor`` apps but every one of +them works in a single-process program too; import via +``from tractor import trionics``. + +.. currentmodule:: tractor.trionics + +Context-manager helpers +----------------------- + +.. autofunction:: gather_contexts + +.. autofunction:: maybe_open_context + +.. autofunction:: maybe_open_nursery + +.. note:: + + :func:`gather_contexts` is "a nursery for async context + managers": it enters N acms concurrently and yields their + values in input order. :func:`maybe_open_context` is the + actor-wide cache/multiplex layer on top — the first task pays + the acm setup cost, later callers get ``(cache_hit=True, ...)`` + and share the same value until all users exit. + +Broadcast fan-out +----------------- + +.. autofunction:: broadcast_receiver + +.. autoclass:: BroadcastReceiver + :members: receive, + subscribe, + aclose + +.. autoexception:: Lagged + :show-inheritance: + +A single-producer, many-consumer broadcast layer over any +``trio``-style receive channel: non-lossy for the *fastest* +consumer while slower consumers raise :class:`Lagged` (a +:class:`trio.TooSlowError` subtype) once they fall behind the +internal ring. This is exactly the machinery behind +:meth:`tractor.MsgStream.subscribe` — see +``examples/streaming_broadcast_fanout.py``. + +ExceptionGroup helpers +---------------------- + +.. autofunction:: collapse_eg + +.. autofunction:: maybe_raise_from_masking_exc + +.. note:: + + :func:`collapse_eg` "un-nests" single-exception + :class:`ExceptionGroup` wrappers from strict-eg ``trio`` + nurseries so your ``except`` clauses match the original error; + :func:`maybe_raise_from_masking_exc` surfaces real errors that + would otherwise be masked by :class:`trio.Cancelled` during + teardown. + +.. seealso:: + + :doc:`/api/context` for the IPC-stream consumer of + :class:`BroadcastReceiver`, :doc:`/guide/streaming` for + fan-out in a worked pipeline, and the `trio docs`_ for the + underlying channel and `nursery`_ semantics these helpers + compose. + +.. _trio docs: https://trio.readthedocs.io/en/latest/ +.. _nursery: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning diff --git a/docs/conf.py b/docs/conf.py index 6bb0ed16d..f1732184e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,105 +1,147 @@ -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html +# tractor: distributed structured concurrency. +''' +Sphinx config for `tractor`'s documentation. -# -- Path setup -------------------------------------------------------------- +Theme-wise we ride the `pydata_sphinx_theme` (per the +research + history in #157) skinned to a minimal +black + white look via `_static/css/custom.css`; see +the local extensions under `_ext/` for our `.. d2::` +diagram and `.. margin::` aside directives. -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# -# import os -# import sys -# sys.path.insert(0, os.path.abspath('.')) +Build locally via, -# Warn about all references to unknown targets -nitpicky = True + uv run --group docs make -C docs html -# The master toctree document. -master_doc = 'index' +''' +from importlib.metadata import version as get_version +from pathlib import Path +import sys -# -- Project information ----------------------------------------------------- +# local sphinx extensions live in `_ext/`: +# - `d2diagrams`: `.. d2::` diagram rendering +# - `marginalia`: `.. margin::` RHS prose-asides +sys.path.insert( + 0, + str((Path(__file__).parent / '_ext').resolve()), +) + +# -- project info --------------------------------- project = 'tractor' -copyright = '2018, Tyler Goodlet' +copyright = '2018-2026, Tyler Goodlet' author = 'Tyler Goodlet' +release: str = get_version('tractor') +version: str = release -# The full version, including alpha/beta/rc tags -release = '0.0.0a0.dev0' - -# -- General configuration --------------------------------------------------- +# -- general config ------------------------------- -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. extensions = [ 'sphinx.ext.autodoc', + 'sphinx.ext.autosummary', 'sphinx.ext.intersphinx', + 'sphinx.ext.viewcode', 'sphinx.ext.todo', + 'sphinx_design', + 'sphinx_copybutton', + 'sphinxext.opengraph', + 'sphinx_togglebutton', + 'd2diagrams', + 'marginalia', ] - -# Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] +exclude_patterns = [ + '_build', + # the pypi/gh readme + its standalone generator + # sub-project; NOT doc-tree pages. + 'README.rst', + 'github_readme', + 'Thumbs.db', + '.DS_Store', +] +root_doc = 'index' +# TODO: flip this on + burn down the (many) warnings +# from our informal docstring style; see the autodoc +# readiness notes from the revamp's recon pass. +nitpicky = False -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +# -- autodoc/autosummary -------------------------- +autodoc_member_order = 'bysource' +autodoc_typehints = 'description' +autosummary_generate = True -# -- Options for HTML output ------------------------------------------------- +# -- intersphinx ---------------------------------- -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = 'sphinx_book_theme' +intersphinx_mapping = { + 'python': ( + 'https://docs.python.org/3', + None, + ), + 'trio': ( + 'https://trio.readthedocs.io/en/stable', + None, + ), + # NOTE, msgspec's site doesn't publish an + # `objects.inv` (404s) so no intersphinx for it. + 'pytest': ( + 'https://docs.pytest.org/en/stable', + None, + ), +} -pygments_style = 'algol_nu' +# -- html output ---------------------------------- -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. +html_theme = 'pydata_sphinx_theme' +html_title = 'tractor' +html_logo = '_static/tractor_logo_side.svg' +html_favicon = '_static/tractor_logo_side.svg' +html_static_path = ['_static'] +html_css_files = ['css/custom.css'] +html_show_sourcelink = False html_theme_options = { - # 'logo': 'tractor_logo_side.svg', - # 'description': 'Structured concurrent "actors"', - "repository_url": "https://github.com/goodboy/tractor", - "use_repository_button": True, - "home_page_in_toc": False, - "show_toc_level": 1, - "path_to_docs": "docs", - + 'logo': { + 'image_light': '_static/tractor_logo_side.svg', + 'image_dark': '_static/tractor_logo_side.svg', + 'alt_text': 'tractor', + }, + 'github_url': 'https://github.com/goodboy/tractor', + 'navbar_align': 'content', + 'show_toc_level': 2, + 'secondary_sidebar_items': { + '**': ['page-toc'], + 'index': [], + }, + 'use_edit_page_button': True, + 'footer_start': ['copyright'], + 'footer_end': ['theme-version'], + 'pygments_light_style': 'algol_nu', + 'pygments_dark_style': 'github-dark', } -html_sidebars = { - "**": [ - "sbt-sidebar-nav.html", - # "sidebar-search-bs.html", - # 'localtoc.html', - ], - # 'logo.html', - # 'github.html', - # 'relations.html', - # 'searchbox.html' - # ] +html_context = { + 'github_user': 'goodboy', + 'github_repo': 'tractor', + 'github_version': 'main', + 'doc_path': 'docs', } -# doesn't seem to work? -# extra_navbar = "

nextttt-gennnnn

" +# -- ext: opengraph ------------------------------- -html_title = '' -html_logo = '_static/tractor_logo_side.svg' -html_favicon = '_static/tractor_logo_side.svg' -# show_navbar_depth = 1 +ogp_site_url = 'https://goodboy.github.io/tractor/' +ogp_use_first_image = True -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# -- ext: copybutton ------------------------------ -# Example configuration for intersphinx: refer to the Python standard library. -intersphinx_mapping = { - "python": ("https://docs.python.org/3", None), - "pytest": ("https://docs.pytest.org/en/latest", None), - "setuptools": ("https://setuptools.readthedocs.io/en/latest", None), -} +copybutton_prompt_text = r'>>> |\.\.\. |\$ ' +copybutton_prompt_is_regexp = True + +# -- ext: todo ------------------------------------ + +todo_include_todos = True + +# -- ext: d2diagrams (local) ---------------------- + +# normally resolved from PATH or the `D2_BIN` env +# var; when neither hits, the committed SVGs under +# `_diagrams/` are used as-is. +d2_bin = None +d2_args = [] diff --git a/docs/dev_tips.rst b/docs/dev_tips.rst deleted file mode 100644 index 93e812dca..000000000 --- a/docs/dev_tips.rst +++ /dev/null @@ -1,51 +0,0 @@ -Hot tips for ``tractor`` hackers -================================ - -This is a WIP guide for newcomers to the project mostly to do with -dev, testing, CI and release gotchas, reminders and best practises. - -``tractor`` is a fairly novel project compared to most since it is -effectively a new way of doing distributed computing in Python and is -much closer to working with an "application level runtime" (like erlang -OTP or scala's akka project) then it is a traditional Python library. -As such, having an arsenal of tools and recipes for figuring out the -right way to debug problems when they do arise is somewhat of -a necessity. - - -Making a Release ----------------- -We currently do nothing special here except the traditional -PyPa release recipe as in `documented by twine`_. I personally -create sub-dirs within the generated `dist/` with an explicit -release name such as `alpha3/` when there's been a sequence of -releases I've made, but it really is up to you how you like to -organize generated sdists locally. - -The resulting build cmds are approximately: - -.. code:: bash - - python setup.py sdist -d ./dist/XXX.X/ - - twine upload -r testpypi dist/XXX.X/* - - twine upload dist/XXX.X/* - - - -.. _documented by twine: https://twine.readthedocs.io/en/latest/#using-twine - - -Debugging and monitoring actor trees ------------------------------------- -TODO: but there are tips in the readme for some terminal commands -which can be used to see the process trees easily on Linux. - - -Using the log system to trace `trio` task flow ----------------------------------------------- -TODO: the logging system is meant to be oriented around -stack "layers" of the runtime such that you can track -"logical abstraction layers" in the code such as errors, cancellation, -IPC and streaming, and the low level transport and wire protocols. diff --git a/docs/diagrams/actor_tree.d2 b/docs/diagrams/actor_tree.d2 new file mode 100644 index 000000000..6bec7a9ed --- /dev/null +++ b/docs/diagrams/actor_tree.d2 @@ -0,0 +1,31 @@ +# tractor docs diagram: the hero supervision tree. +# A process tree of trio-task-trees; every arrow is +# a parent which *must wait* on its children. +vars: { + d2-config: { + sketch: true + theme-id: 1 + pad: 16 + layout-engine: elk + } +} +direction: down +root: "root actor" { + main: "main()" + an: "ActorNursery" + main -> an: opens +} +w0: "subactor\n'worker_0'" { + tasks: "trio task tree" +} +w1: "subactor\n'worker_1'" { + main: "serve()" + an: "ActorNursery" + main -> an: opens +} +gc: "subactor\n'deeper'" { + tasks: "trio task tree" +} +root.an -> w0: "spawns +\nsupervises" +root.an -> w1: "spawns +\nsupervises" +w1.an -> gc: "spawns +\nsupervises" diff --git a/docs/diagrams/context_handshake.d2 b/docs/diagrams/context_handshake.d2 new file mode 100644 index 000000000..119718256 --- /dev/null +++ b/docs/diagrams/context_handshake.d2 @@ -0,0 +1,21 @@ +# tractor docs diagram: the inter-actor `Context` +# dialog; tractor's SC-transitive msg protocol as +# seen from both sides of `Portal.open_context()`. +vars: { + d2-config: { + sketch: true + theme-id: 1 + pad: 16 + layout-engine: elk + } +} +shape: sequence_diagram +parent: "parent task\n(Portal)" +child: "child actor task\n(@tractor.context fn)" +parent -> child: "Start: open_context(fn, **kwargs)" +child -> parent: "StartAck" +child -> parent: "Started: await ctx.started(value)" +parent -> child: "Yield: stream.send()" +child -> parent: "Yield: stream.send()" +parent -> child: "Stop: stream closed" +child -> parent: "Return: fn return value" diff --git a/docs/diagrams/debug_lock.d2 b/docs/diagrams/debug_lock.d2 new file mode 100644 index 000000000..7a96b4301 --- /dev/null +++ b/docs/diagrams/debug_lock.d2 @@ -0,0 +1,19 @@ +# tractor docs diagram: multi-actor debugger REPL +# serialization via the root actor's tty lock. +vars: { + d2-config: { + sketch: true + theme-id: 1 + pad: 16 + layout-engine: elk + } +} +shape: sequence_diagram +mary: "subactor 'mary'" +root: "root actor\n(owns the tty)" +bob: "subactor 'bob'" +mary -> root: "tractor.pause(): acquire tty lock" +root -> mary: "granted: pdb REPL is yours" +bob -> root: "tractor.pause(): acquire tty lock" +mary -> root: "continue: release lock" +root -> bob: "granted: pdb REPL is yours" diff --git a/docs/diagrams/error_propagation.d2 b/docs/diagrams/error_propagation.d2 new file mode 100644 index 000000000..5a956d8f9 --- /dev/null +++ b/docs/diagrams/error_propagation.d2 @@ -0,0 +1,27 @@ +# tractor docs diagram: one-cancels-all error +# propagation up a (sub-)actor tree; no zombies, +# no lost errors. +vars: { + d2-config: { + sketch: true + theme-id: 1 + pad: 16 + layout-engine: elk + } +} +direction: down +root: "root actor\n(ActorNursery)" +gertie: "subactor 'gertie'\n(healthy, gets cancelled)" +bobbie: "subactor 'bobbie'\nraises NameError" +root -> gertie: supervises +root -> bobbie: supervises +bobbie -> root: "RemoteActorError\n(boxed NameError)" { + style: { + stroke-dash: 3 + } +} +root -> gertie: "cancel()" { + style: { + stroke-dash: 3 + } +} diff --git a/docs/diagrams/infected_aio.d2 b/docs/diagrams/infected_aio.d2 new file mode 100644 index 000000000..da7060535 --- /dev/null +++ b/docs/diagrams/infected_aio.d2 @@ -0,0 +1,23 @@ +# tractor docs diagram: "infected asyncio" mode; +# trio runs as a guest on the asyncio loop with +# SC supervision linking tasks across both. +vars: { + d2-config: { + sketch: true + theme-id: 1 + pad: 16 + layout-engine: elk + } +} +direction: down +sub: "subactor process (infect_asyncio=True)" { + aio: "asyncio event loop" { + aiotask: "asyncio.Task\naio_echo_server()" + trio: "trio (guest mode)" { + triotask: "trio task\n@tractor.context fn" + } + trio.triotask <-> aiotask: "LinkedTaskChannel\n.send()/.receive()" + } +} +parent: "parent actor (pure trio)" +parent <-> sub.aio.trio.triotask: "IPC: Context + MsgStream" diff --git a/docs/diagrams/runtime_stack.d2 b/docs/diagrams/runtime_stack.d2 new file mode 100644 index 000000000..72602060c --- /dev/null +++ b/docs/diagrams/runtime_stack.d2 @@ -0,0 +1,16 @@ +# tractor docs diagram: the layered runtime view +# inside any single actor process. +vars: { + d2-config: { + sketch: true + theme-id: 1 + pad: 16 + layout-engine: elk + } +} +grid-rows: 4 +grid-gap: 0 +app: "your app: plain trio tasks + nurseries" +tractor: "tractor runtime: actors, portals,\ncontexts + streams, RPC" +ipc: "IPC Channel: msgspec-typed msgs\nover TCP | UDS transports" +os: "OS: one process per actor" diff --git a/docs/diagrams/streaming_pipeline.d2 b/docs/diagrams/streaming_pipeline.d2 new file mode 100644 index 000000000..1a13ba7be --- /dev/null +++ b/docs/diagrams/streaming_pipeline.d2 @@ -0,0 +1,27 @@ +# tractor docs diagram: multi-actor streaming +# pipeline topology from +# examples/full_fledged_streaming_service.py +vars: { + d2-config: { + sketch: true + theme-id: 1 + pad: 16 + layout-engine: elk + } +} +direction: right +s0: "actor 'streamer_0'" { + fn: "stream_data(0)" +} +s1: "actor 'streamer_1'" { + fn: "stream_data(1)" +} +agg: "actor 'aggregator'" { + fn: "aggregate()" +} +main: "root actor" { + fn: "main()" +} +s0 -> agg: "async for:\nyielded ints" +s1 -> agg: "async for:\nyielded ints" +agg -> main: "deduped\nstream" diff --git a/docs/explain/architecture.rst b/docs/explain/architecture.rst new file mode 100644 index 000000000..37a7f147e --- /dev/null +++ b/docs/explain/architecture.rst @@ -0,0 +1,368 @@ +Anatomy of the runtime +====================== + +You can get a long way with ``tractor`` by treating it as "trio_ +with nurseries that spawn processes". But once you start asking +*where does my msg actually go?*, *which process is that?* or +*who keeps the phonebook?*, it pays to know how the runtime hangs +together. This page walks the stack top to bottom. + +.. d2:: diagrams/runtime_stack.d2 + :caption: The four runtime layers inside *every* actor process. + :alt: layer cake of app tasks, tractor runtime, IPC, OS process + :width: 70% + +The layer cake +-------------- + +Every actor process is the same four-layer sandwich: + +- **your app**: plain ``trio`` tasks, nurseries and cancel + scopes; nothing special. ``tractor`` is a `structured + concurrency`_ (SC) multi-processing runtime built on trio_ and + the whole pitch is that this layer stays *just trio*: no + callbacks, no futures, no proxy objects. +- **the** ``tractor`` **runtime**: a per-process + :class:`tractor.Actor` running the msg loop and RPC task + scheduler, plus the user-facing primitives layered on it: + :class:`tractor.ActorNursery` (spawning + supervision), + :class:`tractor.Portal` (calling into a peer) and + :class:`tractor.Context` + :class:`tractor.MsgStream` + (SC-linked cross-actor task pairs and streaming). +- **IPC channels**: one :class:`tractor.Channel` per connected + peer, each wrapping a ``MsgTransport`` that ships + msgspec_-typed msgs over TCP or UDS. +- **the OS**: one process per actor, started by a swappable + spawn backend. + +The property that holds it all together: SC composes *through* +the layers. A crash in a leaf actor's app task unwinds that +actor's trio tree, ships across its IPC channel as a typed +``Error`` msg, and unwinds the parent's trio tree in turn — the +"SC-transitive supervision protocol" from the README's pitch. +The whole tree cancels and errors like one big trio program; it +just happens to be spread across processes. + +One actor, one process, one ``trio.run()`` +------------------------------------------ + +A ``tractor`` "actor" is not a green thread, nor an object with +a mailbox, nor a coroutine: it's one OS process running one +:func:`trio.run` whose root task boots the runtime machinery — +msg loop, RPC task scheduler, IPC server — all embodied by a +single :class:`tractor.Actor` instance. + +.. margin:: Shared nothing + + Processes buy you a real `shared nothing architecture`_: no + accidentally-shared mutable state, no GIL contention, and + every actor can be inspected (or killed) like any other OS + process. + +You rarely construct an :class:`~tractor.Actor` yourself; the +runtime makes exactly one per process and you grab it with +:func:`tractor.current_actor`: + +.. code:: python + + import tractor + + actor = tractor.current_actor() # NoRuntime if none running + print(actor.aid.name) # str name, need not be unique + print(actor.aid.uuid) # uuid4 str, IS unique + print(actor.aid.pid) # the OS pid + print(actor.uid) # legacy (name, uuid) pair + +Identity is carried by the ``Aid`` msg-struct (see +``tractor.msg.types``): a ``name``/``uuid``/``pid`` triple +exchanged in the very first "mailbox handshake" whenever two +actors connect. It's what the registrar stores and what shows up +in logs and proc-titles. The older ``.uid`` 2-tuple of +``(name, uuid)`` predates ``Aid`` and is still pervasive across +the codebase; treat it as the legacy spelling of the same +identity. + +If this smells like the `actor model`_, sure — but as the +README warns, it probably doesn't look like what you *think* an +actor model looks like, and that's intentional. Here an "actor" +is purely a runtime-unit-of-abstraction: process + +``trio.run()`` + IPC machinery. + +IPC: channels, transports, addresses +------------------------------------ + +Two connected actors talk through a :class:`tractor.Channel`: a +duplex, per-peer msg pipe. Each ``Channel`` wraps a +``MsgTransport`` instance which does the wire work: framing, +encode/decode and the socket itself. The encoding is msgpack +(via msgspec_) and *every* msg is an instance of one of the +runtime's tagged-union :class:`msgspec.Struct` types: the +``Aid`` handshake, ``Start``/``StartAck`` (RPC init), +``Started``/``Yield``/``Stop``/``Return`` (the ctx dialog +phases), ``Error``, etc. There is no raw-bytes mode; the +msg-spec *is* the protocol, which is exactly what lets payloads +be type-limited per-context (see ``pld_spec`` in +:doc:`/guide/context`). + +Addresses come in two spellings: + +- *unwrapped*: the plain-tuple form you pass to user APIs — + ``('127.0.0.1', 1616)`` for tcp, or a + ``(, )`` path-pair for uds; +- *wrapped*: the internal ``TCPAddress``/``UDSAddress`` struct + types (plus libp2p-style multiaddr helpers over in + ``tractor.discovery``). + +You only ever need the tuple form; the runtime wraps and +unwraps at the boundaries. + +TCP: the boring default +*********************** + +The default transport (``'tcp'``) binds each actor's IPC server +to loopback ``('127.0.0.1', )`` unless told +otherwise, and is the only choice when your tree spans hosts. +Nothing exotic: ``trio`` TCP streams + length-prefixed msgpack +framing. + +UDS: same-host, creds included +****************************** + +Pass ``enable_transports=['uds']`` and actors instead talk over +unix-domain sockets, with socket files placed in the per-user +runtime dir (``$XDG_RUNTIME_DIR/tractor/`` on linux, the +``platformdirs`` equivalent elsewhere). Two perks over tcp on a +single host: + +- no ports to fight over; addrs are just file paths, +- the kernel snitches on your peer for free: the listening side + reads the connector's ``pid`` (plus ``uid``/``gid`` on linux) + straight off the socket via ``SO_PEERCRED`` / + ``LOCAL_PEERPID`` — no extra handshake msgs required B) + +.. warning:: + + Socket-file lifetime == listening actor lifetime. On + listener teardown the runtime ``os.unlink()``\s the socket + file immediately, so any *late* connection attempt (say, a + sub-actor racing to deregister with a registrar that's + already shutting down) fails with ``FileNotFoundError``. + And ofc, UDS is same-host only. + +Here's a full actor tree run entirely over uds: + +.. literalinclude:: ../../examples/uds_transport_actor_tree.py + :caption: examples/uds_transport_actor_tree.py + :language: python + +Picking a transport +******************* + +Transport choice is per-actor via the ``enable_transports`` +kwarg accepted by :func:`tractor.open_root_actor` (and proxied +through ``open_nursery()`` when it implicitly boots the +runtime), plus per-child via +``ActorNursery.start_actor(enable_transports=...)``. Two rules +the runtime enforces today: + +- exactly ONE transport per actor: multi-transport actors are + on the roadmap but currently raise ``RuntimeError``; +- your ``registry_addrs`` protos must all be in + ``enable_transports``: mismatches fail fast with + ``ValueError`` instead of (as in darker times) hanging the + registrar handshake forever. + +Spawn backends +-------------- + +How does an actor actually *become* a process? Via a swappable +spawn backend, selected with the ``start_method`` kwarg to +:func:`tractor.open_root_actor`: + +``'trio'`` (default) + The home-grown spawner: re-exec the child as + ``python -m tractor._child`` using ``trio``'s subprocess + machinery, then bootstrap it over the first IPC exchange + (the parent ships a ``SpawnSpec`` msg carrying all init + state). Supported on all platforms and the most battle + tested choice by far. + +``'mp_spawn'`` / ``'mp_forkserver'`` + The stdlib :mod:`multiprocessing` start-methods of the same + names (forkserver is posix-only). Mostly interesting for + ecosystem compat and start-up-latency tuning. + +``'subint'`` (experimental, py3.14+) + Each actor runs as a `PEP 734`_ sub-interpreter + (``concurrent.interpreters``) driven on its own OS thread + *inside the parent process*: interpreter-level + shared-nothing isolation with much faster start-up. Yes, + this bends the one-actor-one-process rule; the rest of the + model is unchanged. + +The ``TRACTOR_SPAWN_METHOD`` env-var beats any caller-passed +``start_method``, so you can swap backends under an unmodified +app: + +.. code:: bash + + TRACTOR_SPAWN_METHOD=mp_forkserver python my_app.py + +One current limitation worth knowing: ``debug_mode=True`` (the +crash-to-REPL machinery) is only supported on backends whose +child-side runtime is trio-native, e.g. the default ``'trio'``; +see :doc:`/guide/debugging` for the deats. + +The registrar +------------- + +Discovery needs a phonebook. Every actor, as part of boot, +registers its ``Aid`` and bind-addrs with the *registrar*: an +otherwise ordinary actor (a :class:`tractor.Registrar`, subtype +of :class:`~tractor.Actor`) that keeps the name -> addrs table +for the tree; on graceful exit each actor de-registers itself. + +.. margin:: Default registry addrs + + With no ``registry_addrs`` passed: + + - tcp: ``('127.0.0.1', 1616)`` + - uds: ``registry@1616.sock`` + in the runtime dir + +Who *is* the registrar? Decided at root boot, rendezvous style. +:func:`tractor.open_root_actor` probes each addr in +``registry_addrs`` with a quick connect-ping, then: + +- **somebody answered**: this root is a plain actor; it + registers with the existing registrar and binds random + same-proto addrs for its own IPC server; +- **nobody answered**: this root *becomes* the registrar and + binds the registry addrs itself. + +So single-program trees need zero config — the root quietly +self-appoints — while multi-program setups share a registrar by +pointing every program at the same ``registry_addrs``. Pass +``ensure_registry=True`` to demand that *this* call create the +registry; it raises if the addrs are already served. + +The lookup APIs — :func:`tractor.find_actor`, +:func:`tractor.wait_for_actor`, :func:`tractor.query_actor` and +:func:`tractor.get_registry` — all consult it (after first +checking already-connected peers): + +.. literalinclude:: ../../examples/service_daemon_discovery.py + :caption: examples/service_daemon_discovery.py + :language: python + +If you bump into "arbiter" in old issues or posts: that's the +legacy name for the same thing, surviving in-code only as the +``Arbiter = Registrar`` class alias; all current terminology is +"registrar"/"registry". Fair warning per the README: this is +still a **very naive** discovery sys (no re-election, no gossip +protocol... yet) and a registrar is expected to outlive its +registrants. + +Runtime env vars +---------------- + +A few env-vars let you re-tune a whole tree *without touching +app code*; each wins over its corresponding kwarg: + +.. list-table:: + :header-rows: 1 + :widths: 30 44 26 + + * - env-var + - effect + - vs. kwarg + * - ``TRACTOR_LOGLEVEL`` + - crank (or silence) console-log verbosity for every actor + in the tree + - beats ``loglevel`` + * - ``TRACTOR_SPAWN_METHOD`` + - swap the process spawn backend + - beats ``start_method`` + * - ``TRACTOR_ENABLE_STACKSCOPE`` + - install the ``SIGUSR1`` task-tree-dump handler in every + actor, even outside ``debug_mode`` (see + :doc:`/guide/debugging`) + - OR'd with ``enable_stack_on_sig`` + +Spotting actors from your shell +******************************* + +Every sub-actor sets an OS-level proc-title of the form +``_subactor[@]`` (via ``setproctitle``, silently +skipped when not installed) so ``ps``/``htop``/``pstree`` show +*which actor is which* at a glance. The README's signature +incantation — watch a tree build and self-destruct live: + +.. code:: bash + + $TERM -e watch -n 0.1 "pstree -a $$" \ + & python examples/nested_actor_tree.py \ + && kill $! + +For scripting there are two stable cmdline markers: + +.. code:: bash + + pgrep -fa '_subactor\[' # live, titled sub-actors + pgrep -fa 'tractor._child' # 'trio'-backend children not + # yet (re)titled + +The title also lands in the kernel ``comm`` (truncated to ~15 +bytes) which survives into zombie state — that's what +``tractor``'s own test-harness reapers key off. To be crystal +clear about the contract though: you should never *need* a +reaper; if you can create zombie child processes (without using +a system signal) it **is a bug** — please report it! + +Logging +------- + +The runtime logs through a thin adapter over stdlib +:mod:`logging` that stamps every record with actor + task info. +Two calls get you going: + +.. code:: python + + from tractor.log import get_console_log, get_logger + + log = get_logger(__name__) # actor/task-aware sub-logger + get_console_log('info') # attach console handler @ level + +(or just pass ``loglevel='info'`` to +:func:`tractor.open_root_actor` and the console handler comes up +with the runtime). + +``tractor`` adds custom levels — and matching logger methods — +that slot between the stdlib ones so you can dial in *which +runtime subsystem* you want to hear from: ``.transport()`` (5), +``.runtime()`` (15), ``.devx()`` (17), ``.cancel()`` (22), plus +a ``PDB`` (500) level for debugger chatter. E.g. +``loglevel='cancel'`` plays the whole cancellation chorus while +staying quiet about transport-layer noise. Beyond that +``tractor`` isn't opinionated about how you consume logs: it's +all stdlib ``logging`` underneath. + +Where to next? +-------------- + +.. seealso:: + + - :doc:`/guide/context` — the SC-linked cross-actor task API + that rides on every ``Channel``. + - :doc:`/guide/debugging` — ``debug_mode``, the multi-process + REPL and ``stackscope`` task-tree dumps. + - :doc:`/explain/sc-distributed` — *why* + one-actor-one-process, and what kind of "actor model" this + is (and isn't). + +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _trio: https://github.com/python-trio/trio +.. _actor model: https://en.wikipedia.org/wiki/Actor_model +.. _shared nothing architecture: https://en.wikipedia.org/wiki/Shared-nothing_architecture +.. _msgspec: https://jcristharif.com/msgspec/ +.. _PEP 734: https://peps.python.org/pep-0734/ diff --git a/docs/explain/index.rst b/docs/explain/index.rst new file mode 100644 index 000000000..110529884 --- /dev/null +++ b/docs/explain/index.rst @@ -0,0 +1,16 @@ +Big ideas +========= +The conceptual core of ``tractor``: what *structured +concurrency* (SC) means once your "tasks" are whole +processes, and how the runtime is layered to deliver +that without (much) magic. + +If you only read one page in these docs make it +:doc:`sc-distributed`; if you read two, follow it with +:doc:`architecture`. + +.. toctree:: + :maxdepth: 2 + + sc-distributed + architecture diff --git a/docs/explain/sc-distributed.rst b/docs/explain/sc-distributed.rst new file mode 100644 index 000000000..86ee1c0dd --- /dev/null +++ b/docs/explain/sc-distributed.rst @@ -0,0 +1,207 @@ +Structured concurrency, across processes +========================================= +``tractor`` makes one bet: the discipline that made +``trio``'s concurrency *readable* — `structured +concurrency`_ (SC) — works just as well when the +"tasks" are whole OS processes talking over a wire. +This page distills what that means, from first +principles, with as little ceremony as possible. + +.. margin:: The canon + + If SC is new to you, the seminal `blog post`_ is + still the best hour you'll spend on concurrent + programming; the `trio docs`_, wikipedia's SC_ + page and the diagrams over at libdill-docs_ round + it out nicely. + +SC in one breath +---------------- +Structured concurrency is the rule that **concurrency +gets a scope**: every task is spawned *inside* a block +(a ``trio`` *nursery*) and that block **cannot exit +until every task it spawned has finished** — returned, +errored, or been cancelled. + +That one rule buys you the properties you already +rely on in sequential code, + +- a function call is a *black box*: when it returns, + everything it started is **done** — no secret + background tasks leaking out the sides, +- an exception **always has somewhere to go**: up the + (task) tree to a parent which is, by construction, + still there waiting, +- cancellation has a well defined *shape*: cancel a + scope and it flows down to every task inside it, + and only those. + +In short: your **runtime task tree matches your source +code's indentation**. Concurrency you can read. + +The leap: process-shaped tasks +------------------------------ +Now swap "task" for "process". + +A ``tractor`` *actor* is just a Python process running +its own ``trio.run()`` — its own private task tree, +sharing **nothing** with its siblings. You spawn +actors from an :class:`tractor.ActorNursery`, which +behaves exactly the way the name implies, + +.. code:: python + + async with tractor.open_nursery() as an: + portal = await an.start_actor( + 'worker', + enable_modules=[__name__], + ) + ... + # ^ block exit == every spawned process has + # completed, errored or been cancelled, and + # been **reaped**. No exceptions, no zombies. + +so the whole program becomes a *tree of process-trees* +— a `supervision tree`_ in erlang-speak — where every +arrow means "spawned by, **waited on by**, and +supervised by". + +.. d2:: diagrams/actor_tree.d2 + :caption: A ``tractor`` program: a process tree of + ``trio`` task trees; every parent **must wait** + on its children. + :width: 85% + +Causality: no process outlives its parent +----------------------------------------- +The stdlib's ``multiprocessing`` (and most "job +queue" systems) treat child processes as +fire-and-forget by default: orphans, zombies, lost +tracebacks and ``kill -9`` cleanup scripts are *your* +problem. ``tractor`` instead inherits ``trio``'s +`causality`_ discipline, + +- **no spawning willy-nilly**: every actor is born + from a nursery block with a known parent, +- **lifetimes nest**: a sub-actor's entire process + tree lives strictly inside its parent's nursery + scope, +- **teardown is guaranteed**: when a scope exits (or + errors, or is cancelled) the runtime SIGINTs, + waits, and (only if it must) hard-kills + reaps + everything underneath. + +We take the zombie thing personally: *if you can +create orphaned child processes without using a +system signal, it* **is a bug** — and there's a test +suite to back that sentence up. + +Errors always propagate (yes, across the wire) +---------------------------------------------- +In ``trio``, an exception in any task tears through +its nursery to a parent that must handle it — +`exceptions always propagate`_. ``tractor`` extends +the same guarantee across process boundaries: an +uncaught error in a remote task is + +1. captured + serialized in the child, +2. shipped home over IPC as a typed ``Error`` msg, +3. re-raised in the parent **boxed** as a + :class:`tractor.RemoteActorError` carrying the + original type (``.boxed_type``), a rendered remote + traceback, and the erroring actor's id, + +while the supervising nursery applies its (currently +*one-cancels-all*, just like ``trio``) strategy to any +sibling actors. A crash three processes deep arrives +at your shell as one coherent, causal traceback chain +— not a silent dead worker and a stuck queue. + +Cancellation is a request, supervision is the rule +-------------------------------------------------- +Cancellation likewise keeps ``trio``'s semantics +*verbatim*, just transported: cancelling an actor +nursery (or a single :class:`tractor.Context` between +two tasks in different processes) sends an explicit +cancel **request** over IPC which the remote runtime +translates into a real ``trio`` cancel-scope cancel — +then *acks back* so the requester can await +confirmation within a bounded time. Nothing is ever +"just killed" first; graceful always precedes brutal. + +Because every cross-process dialog is a pair of +**linked tasks** — one on each side, each inside its +own cancel scope — SC stays *transitive*: supervision +doesn't stop at the process boundary, it tunnels +through every hop of the tree. The wire protocol that +enforces this (a small set of typed msgs: +``Start``/``Started``/``Yield``/``Stop``/``Return``/ +``Error``) is detailed in :doc:`/guide/msging` and +:doc:`/guide/context`. + +Hold up, is this an "actor model"? +---------------------------------- +Let's stop and ask how many canon actor model papers +you've actually read ;) + +From `the author's mouth`_, the **only** requirement +is `adherence to`_ the `3 axioms`_:: + + In response to a message, an actor may: + + - send a finite number of new messages + - create a finite number of new actors + - designate a new behavior to process subsequent + messages + +``tractor`` adheres — actors exchange msgs, spawn +actors, and swap behaviors — **with no extra API** to +learn. What we *don't* copy is the cultural baggage: +no visible mailboxes, no untyped fire-and-forget +``send()``, no "let it crash" without a supervisor +that actually hears about it, and definitely no +shared-reference *proxy objects* pretending the +network isn't there. If our "actors" don't look like +what you expected, that's **intentional**: being an +actor model is just one property of the system; being +*structured* is the point. + +Why processes at all? +--------------------- +Python has a GIL; an actor model by definition shares +no state; so the *process* is the natural runtime +unit — you get real multi-core parallelism and hard +memory isolation for free. But the deeper win is +uniformity: because actors only ever talk via msgs +over a :class:`tractor.Channel` (TCP, UDS, more to +come), the **same code** runs your laptop's worker +pool and a multi-host cluster; "distributed" is a +deployment detail, not an API. + +It's just ``trio`` +------------------ +If you remember one framing, make it this: ``tractor`` +**is just** ``trio`` — with nurseries that can spawn +processes and streams that can cross them. Same +nursery discipline, same cancellation semantics, same +"how was this not always the API?" feeling, one level +up the process tree. + +.. seealso:: + + :doc:`/explain/architecture` for how the runtime + layers deliver all of the above, and + :doc:`/start/quickstart` to feel it in ~20 lines of + code. + +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _SC: https://en.wikipedia.org/wiki/Structured_concurrency +.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ +.. _trio docs: https://trio.readthedocs.io/en/latest/ +.. _libdill-docs: https://sustrik.github.io/libdill/structured-concurrency.html +.. _supervision tree: https://www.erlang.org/doc/design_principles/des_princ.html +.. _causality: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#c-c-c-c-causality-breaker +.. _exceptions always propagate: https://trio.readthedocs.io/en/latest/design.html#exceptions-always-propagate +.. _the author's mouth: https://www.youtube.com/watch?v=7erJ1DV_Tlo&t=162s +.. _adherence to: https://www.youtube.com/watch?v=7erJ1DV_Tlo&t=1821s +.. _3 axioms: https://en.wikipedia.org/wiki/Actor_model#Fundamental_concepts diff --git a/docs/guide/asyncio.rst b/docs/guide/asyncio.rst new file mode 100644 index 000000000..27e1d05fd --- /dev/null +++ b/docs/guide/asyncio.rst @@ -0,0 +1,309 @@ +Infected ``asyncio`` +==================== +``tractor`` is "just trio_", but the Python world is packed with +libraries that only speak ``asyncio``: websocket stacks, vendor +SDKs, that one exchange client you can't route around. Rather than +make you rewrite them, ``tractor`` lets you *quarantine* them inside +a dedicated subactor which runs both event loops at once, with full +`structured concurrency`_ (SC) guarantees maintained across the +loop boundary *and* the process tree. + +In the project's own words: + + Yes, we spawn a python process, run ``asyncio``, start ``trio`` + on the ``asyncio`` loop, then send commands to the ``trio`` + scheduled tasks to tell ``asyncio`` tasks what to do XD + +We call this "infected ``asyncio``" mode: the subactor's stdlib +loop runs as the *host* with ``trio`` embedded on top in `guest`_ +mode, and your ``trio`` tasks drive ``asyncio`` tasks through +a linked, SC-supervised, in-memory channel. + +.. d2:: diagrams/infected_aio.d2 + :caption: One process, two schedulers: ``trio`` rides the + ``asyncio`` loop as a guest while the parent speaks plain + ``tractor`` IPC, none the wiser. + :alt: parent actor connected over IPC to a subactor whose + asyncio loop hosts trio in guest mode, with a + LinkedTaskChannel pairing a trio task to an asyncio task + +.. note:: + + Infected ``asyncio`` mode is **experimental**: it works (we + beat on it plenty) but parts of the API surface and some + edge-case semantics are still settling. Got opinions on the + interop design? Feel free to sling them in `#273`_! + +How the infection takes hold +---------------------------- +A normal subactor boots by running the ``tractor`` runtime's task +tree directly under ``trio.run()``. Pass ``infect_asyncio=True`` +at spawn time and the child's entrypoint changes shape entirely: + +1. the process starts the stdlib loop via ``asyncio.run()``, +2. the first ``asyncio`` task calls + ``trio.lowlevel.start_guest_run()``, embedding the ``trio`` + scheduler *inside* the already running ``asyncio`` loop (the + upstream `guest`_-mode feature), +3. the regular ``tractor`` runtime then boots on the guest + ``trio`` side and connects back to its parent like any other + subactor. + +.. margin:: Symptoms + + Looks like your stdlib event loop has caught a case of "the + trios"! Don't worry, you'll barely notice; and if anything + gets too bad, your parents will know about it B) + +Both schedulers interleave in a single thread, no GIL gymnastics +required. From the rest of the actor tree the infected child is +indistinguishable from any other actor: same IPC protocol, same +supervision and cancellation semantics, same zombie-safety +guarantees. The difference is purely internal: ``trio`` tasks in +that process can start and drive ``asyncio`` tasks through the +``tractor.to_asyncio`` API. + +Spawning an infected subactor +----------------------------- +Just flip the flag on :meth:`tractor.ActorNursery.start_actor`: + +.. code:: python + + async with tractor.open_nursery() as an: + portal = await an.start_actor( + 'aio_side', + enable_modules=[__name__], + infect_asyncio=True, + ) + +The one-shot convenience ``ActorNursery.run_in_actor()`` accepts +the same flag. The ``to_asyncio`` APIs may **only** be called from +tasks inside an infected actor; calling them anywhere else raises +a loud ``RuntimeError``. You can introspect at runtime with +``tractor.current_actor().is_infected_aio()``. + +Linking tasks with ``open_channel_from()`` +------------------------------------------ +The core primitive is :func:`tractor.to_asyncio.open_channel_from`, +an async context manager which starts your ``asyncio`` function as +a real ``asyncio.Task`` and yields a two-way channel linking it to +the calling ``trio`` task: + +.. code:: python + + from tractor import to_asyncio + + async with to_asyncio.open_channel_from( + aio_main, # async def aio_main(chan, **kwargs) + period=0.5, # extra kwargs are passed through + ) as (chan, first): + await chan.send('tick') + +The semantics deliberately mirror the inter-actor ``Context`` +handshake from :doc:`/guide/context`: + +- the target fn must declare a parameter literally named ``chan``; + the runtime injects the shared + :class:`~tractor.to_asyncio.LinkedTaskChannel` by keyword. +- the ``trio`` side blocks at entry until the ``asyncio`` task + calls ``chan.started_nowait(value)``; that value is delivered as + ``first``, exactly like the ``(ctx, first)`` pair you get from + ``Portal.open_context()`` after the child calls + ``ctx.started()``. +- a first value **must** be sent from the ``asyncio`` side or the + ``trio`` side will never unblock. +- on block exit the pair is torn down *together*; neither task can + outlive the other (more on this below). + +A full example: the echo server +------------------------------- +Here's the canonical demo, a round-trip echo service where the +``asyncio`` task is told what to do by a ``trio`` task which is in +turn driven over IPC by the root actor: + +.. literalinclude:: ../../examples/infected_asyncio_echo_server.py + :caption: examples/infected_asyncio_echo_server.py + :language: python + +What's going on? + +- there are three task layers: the root actor's pure ``trio`` + task, the infected child's ``trio``-side ``@tractor.context`` + endpoint (``trio_to_aio_echo_server()``), and the child's + ``asyncio`` task (``aio_echo_server()``). +- two ``started``-style handshakes compose: the aio task's + ``chan.started_nowait('start')`` unblocks the child's + ``open_channel_from()`` entry, then the child relays that same + value up via ``await ctx.started(first)`` which unblocks the + root's ``open_context()`` entry. Synchronization all the way + down, er, up. +- each round trip flows: root ``stream.send()`` -> IPC -> child + ``async for msg in stream`` -> ``chan.send(msg)`` -> aio + ``await chan.get()`` -> ``chan.send_nowait()`` -> child + ``chan.receive()`` -> ``stream.send(out)`` -> IPC -> root. +- when the root breaks out of its stream loop and exits the + context block, the child's stream ends, its channel block exits, + and the ``asyncio`` task is reaped along with it; the final + ``portal.cancel_actor()`` then tears down the whole process. No + orphaned ``asyncio`` tasks, no zombie procs; if you manage to + create either it **is a bug**. + +``LinkedTaskChannel``: one channel, two sides +--------------------------------------------- +The same channel object is shared by both tasks; which methods you +call depends on which loop schedules your task. The ``trio`` side +gets a standard ``trio.abc.Channel`` interface while the +``asyncio`` side gets queue-flavored, mostly-sync methods: + +.. list-table:: + :header-rows: 1 + :widths: 14 36 50 + + * - side + - call + - what it does + * - ``trio`` + - ``await chan.send(item)`` + - ship ``item`` to the ``asyncio`` task (enqueues onto an + internal ``asyncio.Queue``). + * - ``trio`` + - ``await chan.receive()`` + - wait for the next value from the ``asyncio`` side; the + channel also supports ``async for``. + * - ``trio`` + - ``await chan.wait_for_result()`` + - block until the ``asyncio`` task completes; return its + final result or raise its (translated) error. + * - ``trio`` + - ``chan.subscribe()`` + - acm yielding a ``BroadcastReceiver`` so N local tasks can + each consume a copy of the inbound stream (see below). + * - ``trio`` + - ``chan.cancel_asyncio_task()`` + - explicitly request cancellation of the linked ``asyncio`` + task. + * - ``asyncio`` + - ``chan.started_nowait(value)`` + - deliver the "first" value; unblocks the ``trio`` side's + ``open_channel_from()`` entry (mirrors ``ctx.started()``). + * - ``asyncio`` + - ``await chan.get()`` + - wait for the next value sent from the ``trio`` side. + * - ``asyncio`` + - ``chan.send_nowait(item)`` + - push a value to the ``trio`` side without blocking. + +Fan-out with ``.subscribe()`` +***************************** +Just like :meth:`tractor.MsgStream.subscribe` does for IPC +streams, ``chan.subscribe()`` lets multiple local ``trio`` tasks +each receive *every* value sent from the single ``asyncio`` task: + +.. code:: python + + async with chan.subscribe() as bcast: + async for msg in bcast: + ... + +The underlying broadcast machinery is lazily allocated on first +use and is *not* reversible for the channel's remaining lifetime, +so only reach for it when you actually want the fan-out. + +One-shot calls with ``run_task()`` +---------------------------------- +When you just want a single ``asyncio`` result and no streaming +dialog, skip the channel ceremony and use +:func:`tractor.to_asyncio.run_task`: + +.. code:: python + + import asyncio + from tractor import to_asyncio + + async def aio_fetch(url: str) -> str: + await asyncio.sleep(0.3) # pretend-IO, aio style + return f'sup {url}' + + # from any trio task inside the infected actor: + page = await to_asyncio.run_task(aio_fetch, url='https://x.io') + +It schedules the fn as an ``asyncio.Task``, waits for completion +and hands the return value back to ``trio``; think of it as the +cross-loop sibling of ``ActorNursery.run_in_actor()``. Errors and +cancellation are translated exactly as for channels. + +Cross-loop errors and cancellation +---------------------------------- +The paired tasks are *SC linked*: exception and cancel handling +tears down **both** sides on any unexpected error or cancellation, +in either loop. There is no fire-and-forget mode; a +``LinkedTaskChannel`` is a supervision scope just like a +``Context`` is across processes. + +Because each loop has its own (incompatible) cancellation and exit +machinery, boundary crossings are translated into dedicated +exception types, all importable from ``tractor.to_asyncio``: + +.. list-table:: + :header-rows: 1 + :widths: 26 22 52 + + * - exception + - raised in + - meaning + * - ``AsyncioCancelled`` + - the ``trio`` task + - the linked ``asyncio`` task was cancelled by itself or + a 3rd party (i.e. *not* by the ``trio`` side). + * - ``AsyncioTaskExited`` + - the ``trio`` task + - the ``asyncio`` task returned/exited early while the + ``trio`` side still held the link open. + * - ``TrioCancelled`` + - the ``asyncio`` task + - the ``trio`` side was cancelled (or crashed) so the + ``asyncio`` task is being torn down per SC rules. + * - ``TrioTaskExited`` + - the ``asyncio`` task + - the ``trio`` side exited gracefully while the ``asyncio`` + task was still running; a "clean shutdown" signal much + like closing a ``trio`` mem-chan. + +By default ``open_channel_from(suppress_graceful_exits=True)`` +absorbs the two ``*TaskExited`` signals so happy-path teardown +stays silent; pass ``False`` when your app wants to handle early +peer-exit explicitly. + +Past the task pair, everything composes with the normal actor +story: an unhandled ``asyncio`` error is translated into the +``trio`` side, propagates out of your ``@tractor.context`` +endpoint, and arrives at the parent boxed as +a :class:`tractor.RemoteActorError`. One SC discipline, +end-to-end, across loops *and* processes. + +Breakpoints in ``asyncio`` tasks +-------------------------------- +Yes, the multi-actor REPL works here too. With +``debug_mode=True`` enabled on your tree the ``trio`` side of an +infected actor can ``await tractor.pause()`` as usual, and with +greenback enabled (``maybe_enable_greenback=True``) even the +builtin ``breakpoint()`` works from *inside* ``asyncio`` tasks; +see ``examples/debugging/asyncio_bp.py`` for the full tour. The +root-TTY locking dance behind all this is covered in +:doc:`/guide/debugging`. + +Where to next? +-------------- +.. seealso:: + + - :doc:`/guide/context` for the inter-actor handshake and + streaming APIs which this whole interop layer mirrors. + - :doc:`/guide/msging` for typing the payloads you shuttle + between actors (and loops). + - :doc:`/guide/debugging` for the multi-process REPL that + keeps working even when your loop has "the trios". + +.. _trio: https://github.com/python-trio/trio +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _guest: https://trio.readthedocs.io/en/stable/reference-lowlevel.html?highlight=guest%20mode#using-guest-mode-to-run-trio-on-top-of-other-event-loops +.. _#273: https://github.com/goodboy/tractor/issues/273 diff --git a/docs/guide/cancellation.rst b/docs/guide/cancellation.rst new file mode 100644 index 000000000..56f0d6d12 --- /dev/null +++ b/docs/guide/cancellation.rst @@ -0,0 +1,285 @@ +Cancellation and error propagation +================================== + +``tractor`` supports ``trio``'s cancellation_ system *verbatim*, +then extends it across process boundaries. If you know how to +cancel a task in ``trio`` you already know how to cancel an actor — +and its whole subtree — in ``tractor``; the runtime's job is making +that statement hold over IPC with every structured concurrency (SC) +guarantee intact. + +The ground rules, + +- a remote actor is **never** cancelled unless explicitly requested + (by a parent or peer), unless supervision demands it (an error + triggered one-cancels-all teardown), or unless there's a bug in + ``tractor`` itself (please report it!), +- (remote) errors `always propagate`_ back to the parent + supervisor; nothing is silently dropped on the floor, +- every spawned process gets reaped no matter how it dies; if you + can create a zombie child process (without using a system signal) + it **is a bug**. + +``trio`` cancellation, across the wire +-------------------------------------- + +Locally everything is bog-standard ``trio``: nurseries, cancel +scopes, timeouts. ``tractor`` adds exactly one twist: a cancel +scope can't physically reach into another process, so the runtime +*relays cancellation as messages*. Concretely, + +- cancelling an *actor* means sending it a runtime-cancel request + msg; the target then runs its own graceful teardown — cancelling + RPC tasks, closing channels, exiting its :func:`trio.run` — and + acks the request back to the canceller, +- cancelling a single *cross-actor task* works through the + :class:`tractor.Context` layer: each ``ctx`` task-pair is + cancel-scope-linked over IPC such that either side erroring or + cancelling relays an equivalent error to the other side (see + :doc:`/guide/context` for the gory details), +- a cancel is therefore always a *request with an ack*: the + canceller does a **bounded wait** for confirmation and escalates + if the peer is unresponsive (see the teardown ladder below). + +One-cancels-all supervision +--------------------------- + +An :class:`tractor.ActorNursery` supervises subactors `exactly like +trio`_ nurseries supervise tasks: when one child errors, the error +propagates to the supervising block and **all** sibling subactors +get cancelled before the error continues bubbling up the (process) +tree. + +.. d2:: diagrams/error_propagation.d2 + :caption: One-cancels-all: no zombies, no lost errors. + :alt: error propagation up a subactor tree + :width: 80% + +.. literalinclude:: ../../examples/remote_error_propagation.py + :caption: examples/remote_error_propagation.py + :language: python + +What's going on here? + +- three healthy actors are spawned as daemons via + :meth:`tractor.ActorNursery.start_actor`; left alone they'd + happily idle forever, +- a fourth actor runs ``assert_err()`` via ``.run_in_actor()`` and + promptly trips its ``assert 0``, +- the resulting ``AssertionError`` ships back over IPC as a + serialized error msg and re-raises *boxed* inside the nursery + block as a :class:`tractor.RemoteActorError`, +- the nursery reacts like any ``trio`` nursery would: it cancels + the three healthy siblings (graceful runtime-cancel requests, + acks awaited), reaps all four processes, then re-raises, +- ``trio.run(main)`` sees that same ``RemoteActorError`` in the + parent-most process — propagation is end-to-end or bust. + +This one-cancels-all style is currently the *only* supervision +strategy offered (it's the one ``trio`` gives you); more +`erlang strategies`_ are roadmap, see the bottom of this page. + +The boxed-error bestiary +------------------------ + +All remote failures arrive locally as one of a small set of +exception types, each carrying enough metadata to work out *who* +failed, *where*, and *why*. + +``RemoteActorError`` +******************** + +The workhorse: a "boxed" exception relayed over IPC from another +actor. The original error's type, traceback string and msgdata are +preserved so you can pattern-match on what actually went wrong +remotely, + +- ``.boxed_type``: the reconstructed **type** of the original + remote exception (``ValueError``, ``NameError``, what have you), +- ``.src_uid``: the ``(name, uuid)`` pair of the actor where the + error *originated*, +- ``.relay_uid`` / ``.relay_path``: when an error crosses more than + one actor boundary (grandchild -> child -> root) every relaying + actor is recorded; multi-hop boxings are lovingly referred to as + "inceptions" in the runtime internals, +- ``.pformat()``: a rich "tb box" rendering of the remote traceback + for your logs or REPL. + +.. code:: python + + try: + async with portal.open_context(ep) as (ctx, first): + ... + except tractor.RemoteActorError as rae: + if rae.boxed_type is ValueError: + ... # the remote task raised `ValueError` + +``ContextCancelled`` +******************** + +The cancel-ack for a cross-actor task pair: raised when a +:class:`tractor.Context` task is cancelled *by request*. Its +``.canceller`` attr is the uid of the actor which **requested** the +cancel, which powers the key rule, + +- if **you** requested it (you called + :meth:`tractor.Context.cancel`) the resulting ctxc is *absorbed* + at ``open_context()`` exit: an expected outcome, not an error, +- if **anyone else** did — the peer task, or some third-party actor + — it *raises* locally so your code always hears about it. + +The full self- vs. cross-cancel semantics are a core teaching point +of :doc:`/guide/context`; go read them there. + +``MsgTypeError`` +**************** + +An IPC-payload "type error": a msg violated the dialog's declared +payload spec. See :doc:`/guide/msging` for the typed-messaging +system which enforces it. + +``TransportClosed`` +******************* + +The underlying IPC transport (TCP stream, UDS socket, ...) died or +closed out from under a channel. You'll normally only see this +surface when a peer hard-exits without any graceful runtime +teardown; the supervision machinery treats unexpected transport +loss on a busy channel as a failure and tears down accordingly. + +Pick your blast radius +---------------------- + +Three cancel surfaces, three scopes of effect; choose the smallest +hammer that does the job. + +.. list-table:: + :header-rows: 1 + :widths: 36 34 30 + + * - surface + - cancels + - typical use + * - :meth:`tractor.ActorNursery.cancel` + - every subactor in the nursery + - whole-tree teardown + * - :meth:`tractor.Portal.cancel_actor` + - one actor: full runtime + proc + - daemon teardown + * - :meth:`tractor.Context.cancel` + - exactly one remote task + - surgical task cancel + +``ActorNursery.cancel()`` +************************* + +The big red button: gracefully cancel every subactor supervised by +the nursery, in parallel, with the escalation discipline below +applied per-child. It's invoked for you whenever an error hits the +nursery block (one-cancels-all); call it yourself for an orderly +early shutdown. Passing ``hard_kill=True`` skips the graceful phase +and goes straight to OS-level process termination — rarely what you +want outside tests. + +``Portal.cancel_actor()`` +************************* + +Cancel one **whole actor**: its entire runtime, every task it's +scheduled, and (for subactors) the OS process, via a graceful +runtime-cancel request, + +.. code:: python + + await portal.cancel_actor() # bounded wait, bool result + await portal.cancel_actor( + raise_on_timeout=True, # no ack in time? + ) # -> `ActorTooSlowError` + +The wait for the peer's ack is *bounded* (default +``Portal.cancel_timeout = 0.5`` seconds, tunable per call via +``timeout=``). By default a missed ack just returns ``False``; with +``raise_on_timeout=True`` you instead get an ``ActorTooSlowError`` +(from ``tractor._exceptions``) so *your* code can escalate per SC +discipline — exactly what the nursery's own teardown does +internally before resorting to OS-level signalling. + +Note the granularity: this cancels an **actor**, not a task. For +one remote task use the ``Context`` layer instead. + +``Context.cancel()`` +******************** + +Request cancellation of exactly one remote task: the peer task of +an open :class:`tractor.Context`. Two things to keep straight, + +- it cancels the **remote** side only; a ``Context`` is *not* a + :class:`trio.CancelScope` and your local task keeps running until + you exit the ``open_context()`` block, +- the resulting :class:`tractor.ContextCancelled` is absorbed + locally (you asked for it, after all) per the self- vs. + cross-cancel rule above. + +Again, :doc:`/guide/context` covers this dance in depth. + +Graceful first, hard as a last resort +------------------------------------- + +.. margin:: REPL-safe by design + + The hard-kill path is *skipped* whenever an actor in the tree + holds the debug-REPL lock (``debug_mode=True`` flavors): + SIGTERM raining down on a tree mid-``pdb`` session would + clobber your prompt. See :doc:`/guide/debugging`. + +Every process teardown in ``tractor`` walks the same escalation +ladder, top rung first, + +1. **graceful cancel request**: a runtime-cancel msg over IPC; the + target actor cancels its tasks, closes its channels and exits + its :func:`trio.run` cleanly, +2. **soft wait**: the parent waits (bounded) for the child process + to exit on its own, +3. **SIGTERM**: no ack within the bounded wait (internally an + ``ActorTooSlowError``) escalates to ``proc.terminate()``, +4. **SIGKILL ultimatum**: still alive after the hard-kill timeout + (~1.6s)? The runtime logs that the "T-800" has been deployed to + collect the zombie and issues ``proc.kill()``. No survivors. + +The result is the **no-zombies guarantee**: ``tractor`` tries to +protect you from zombies, no matter what. Quoting the project +manifesto, + + If you can create zombie child processes (without using + a system signal) it **is a bug**. + +Run the quickstart's self-destructing process-tree demo +(``examples/parallelism/we_are_processes.py``, walked through in +:doc:`/start/quickstart`) under a ``pstree`` watcher and try to +catch a +straggler; we'll wait B) + +Roadmap: ``erlang``-style strategies +------------------------------------ + +One-cancels-all is ``trio``'s strategy and, for now, the only one +``tractor`` ships. Pluggable `erlang strategies`_ — one-for-one +restarts, rest-for-one, transient/permanent child specs and friends +(see the `supervision strategies`_ canon) — are a long-standing +roadmap item tracked in `#22`_. If supervisors are your jam that +issue is the place to sling opinions. + +.. seealso:: + + - :doc:`/guide/context` — the cross-actor task layer where + per-task cancellation actually lives, + - :doc:`/guide/msging` — the typed msg layer that raises + :class:`tractor.MsgTypeError`, + - :doc:`/guide/debugging` — what cancellation does (and very + carefully does *not* do) while a REPL is up. + +.. _cancellation: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-and-timeouts +.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics +.. _always propagate: https://trio.readthedocs.io/en/latest/design.html#exceptions-always-propagate +.. _erlang strategies: https://learnyousomeerlang.com/supervisors +.. _supervision strategies: https://www.erlang.org/doc/system/sup_princ.html +.. _#22: https://github.com/goodboy/tractor/issues/22 diff --git a/docs/guide/clustering.rst b/docs/guide/clustering.rst new file mode 100644 index 000000000..bc56b1afc --- /dev/null +++ b/docs/guide/clustering.rst @@ -0,0 +1,127 @@ +Higher-level cluster APIs +========================= + +Sometimes you don't want a hand-crafted supervision tree; you want +"a pile of workers, one per core, now please". For that there's +:func:`tractor.open_actor_cluster`: a convenience wrapper which +spawns a *flat* cluster of subactors and hands you back a portal to +each, + +.. code:: python + + @acm + async def open_actor_cluster( + modules: list[str], # RPC allowlist for workers + count: int = cpu_count(), # one per core by default + names: list[str]|None = None, # default: 'worker_{i}' + hard_kill: bool = False, # fwd to `an.cancel()` + **runtime_kwargs, # fwd to `open_root_actor()` + ) -> AsyncGenerator[dict[str, tractor.Portal], None]: + +A cluster in one block +---------------------- + +.. literalinclude:: ../../examples/quick_cluster.py + :caption: examples/quick_cluster.py + :language: python + +Walkthrough, + +- ``open_actor_cluster(modules=[__name__])`` concurrently spawns + one subactor per detected core (per + :func:`multiprocessing.cpu_count`); the ``modules`` list is the + usual ``enable_modules``-style capability allowlist so workers + may run functions defined in this module, +- it yields a ``dict[str, tractor.Portal]`` mapping worker name to + portal; note the keys get prefixed with the *spawning* actor's + name, so from the root you'll see ``'root.worker_0'``, + ``'root.worker_1'``, etc., +- a plain :class:`trio.Nursery` then fans out one + ``portal.run(sleepy_jane)`` per worker; each prints its actor + ``.uid`` from inside its own process then naps forever — what + runs *inside* each worker (and how many tasks you point at it) + is entirely yours to compose, +- ``tractor.trionics.collapse_eg()`` un-nests the strict + ``ExceptionGroup`` wrapping so the demo's ``KeyboardInterrupt`` + surfaces as itself instead of arriving eg-boxed, +- on block exit the whole fleet is torn down for you via + :meth:`tractor.ActorNursery.cancel`; pass ``hard_kill=True`` at + open time to skip straight to OS-level termination instead of + the graceful ladder described in :doc:`/guide/cancellation`. + +Sizing, naming, fleet-wide options +---------------------------------- + +``count`` doesn't have to be core-count and the auto-generated +``'worker_{i}'`` names are just the default; pass your own (the +length must match ``count`` or you get a ``ValueError``). Any +extra ``**runtime_kwargs`` pass through verbatim to +:func:`tractor.open_root_actor`, so fleet-wide runtime options are +one kwarg away, + +.. code:: python + + async with tractor.open_actor_cluster( + modules=['mylib.workers'], + count=4, + names=['scout', 'miner', 'smelter', 'smith'], + debug_mode=True, # whole-fleet crash-to-REPL + ) as portal_map: + ... + +From here the composition patterns are the usual ``tractor`` fare: +``portal.run()`` for one-shot calls (as in the demo), or — for a +persistent bidirectional dialog per worker — concurrently enter N +``portal.open_context()`` blocks with +``tractor.trionics.gather_contexts()``; see :doc:`/guide/context` +for that whole layer. + +Clusters vs. nurseries +---------------------- + +.. d2:: diagrams/actor_tree.d2 + :margin: + :caption: The general shape: arbitrary nesting. A cluster is + this, minus the nesting. + :alt: a nested supervision tree of subactors + +``open_actor_cluster()`` is sugar, not a new primitive: under the +hood it's just :func:`tractor.open_nursery` plus N concurrent +``start_actor()`` calls plus a ``.cancel()`` on the way out. Reach +for it when, + +- you want a *flat*, homogeneous fleet (classic worker-pool or + map-style fan-out shapes), +- "one per core" — or a fixed ``count`` — is the right sizing + story, +- every child can share the same spawn options. + +Drop down to a raw :class:`tractor.ActorNursery` when the topology +gets any fancier: nested trees, heterogeneous children, per-child +``debug_mode``/transport/module options, daemons mixed with +one-shot workers, and so on (see :doc:`/guide/parallelism` for a +hand-rolled pool). Either way the supervision semantics are +identical: one-cancels-all error propagation and the no-zombies +guarantee from :doc:`/guide/cancellation` apply to clusters too. + +Provisional, by design +---------------------- + +.. note:: + + APIs in this section are considered **provisional**: the + signature and semantics of :func:`tractor.open_actor_cluster` + may shift as higher-level supervision machinery lands. We + encourage you to try it and provide feedback — the + `matrix channel`_ is the place to say hi, and `#22`_ tracks the + broader supervisor-strategy roadmap. + +.. seealso:: + + - :doc:`/guide/parallelism` — worker pools built "by hand" with + plain actor nurseries (and why that's easy peasy), + - :doc:`/guide/cancellation` — the teardown machinery a cluster + inherits for free. + +.. _matrix channel: https://matrix.to/#/!tractor:matrix.org +.. _#22: https://github.com/goodboy/tractor/issues/22 diff --git a/docs/guide/context.rst b/docs/guide/context.rst new file mode 100644 index 000000000..9ecab5546 --- /dev/null +++ b/docs/guide/context.rst @@ -0,0 +1,344 @@ +The ``Context``: a cross-actor task pair +========================================= + +If you've written any trio_ you already know the contract: every +task lives in a nursery, errors always propagate, cancellation is +scoped, and nothing leaks. ``tractor`` extends that exact contract +*across processes* — the same guarantees from the seminal +`blog post`_, just with the nursery split across two memory +domains. The primitive that does it is :class:`tractor.Context`: a +**linked pair of tasks**, one in each of two actors, supervised as +a single `structured concurrency`_ (SC) scope over IPC. + +.. d2:: diagrams/context_handshake.d2 + :caption: The SC-transitive supervision protocol, msg by msg. + :alt: sequence diagram of the context handshake msg flow + +Pretty much everything else is (or is slated to be) built on this +one primitive: ``ActorNursery.run_in_actor()`` is a convenience +for "spawn, open a context, await the result, tear down"; plain +``Portal.run()`` RPC is planned to be re-implemented on top of it; +the multi-process debugger's tree-wide REPL lock rides one. Grok +this page and the rest of the library reads as convenience +wrappers B) + +The endpoint contract +--------------------- + +A context endpoint is an async function decorated with +:func:`tractor.context` which declares **a param annotated** +``tractor.Context`` — any param name you like, the annotation is +what's required: + +.. code:: python + + @tractor.context + async def trainer( + ctx: tractor.Context, + model: str, + ) -> str: + await ctx.started('ready') + return f'trained {model}' + +.. margin:: Who am I talking to? + + Inside any context task + :func:`tractor.current_ipc_ctx` returns the + ``Context`` bound to the current task; handy + in helpers that don't take ``ctx`` explicitly. + +The parent (aka "opener") side invokes it through a +:class:`tractor.Portal` using ``Portal.open_context()``, passing +any extra kwargs which are shipped over the wire as the remote +task's arguments. Since the target fn is referenced by module +path, that module must be listed in the peer actor's +``enable_modules`` allowlist — RPC capability is always opt-in. + +The decorator also accepts a ``pld_spec``: a type (union) which +every payload in the dialog is validated against, upgrading your +msgs to a typed contract enforced via :exc:`tractor.MsgTypeError`. +Validation strictness follows the "`cheap or nasty`_" +`(un)protocol`_ pattern: the one-shot ``Started`` payload gets the +nasty treatment (stringently round-trip checked before it's even +sent) while high-rate stream payloads stay cheap (checked only +receiver side). + +The handshake, on the wire +-------------------------- + +Every context runs one instance of ``tractor``'s "SC-transitive +supervision protocol": a tiny fixed grammar of msgspec_-typed msgs +encapsulating *all* RPC dialogs between actors. *Transitive* +because each IPC link obeys the same rules a local nursery does — +starts are acked, completion is awaited, errors and cancels always +relay — so chaining links across a process tree composes into one +tree-wide SC scope. + +The figure up top shows a full dialog; in order: + +``Start`` + sent by ``Portal.open_context()``: "schedule a task running + this function with these kwargs". + +``StartAck`` + the peer runtime confirms the task is scheduled and that the + endpoint really is a context-style fn. + +``Started`` + emitted when the child task calls + :meth:`tractor.Context.started`; carries the first payload + and unblocks the parent's entry of ``open_context()``. + +``Yield`` + one per :meth:`tractor.MsgStream.send`, flowing in *either* + direction while a stream is open. + +``Stop`` + graceful end-of-stream: the far side's ``async for`` + terminates cleanly. + +``Return`` + the child fn returned; its value becomes the context's final + result. If the child raised instead, an ``Error`` msg takes + this slot carrying the boxed traceback. + +``ctx.started()``: just like ``task_status.started()`` +******************************************************* + +The startup phase is a deliberate clone of +:meth:`trio.Nursery.start` semantics: the child decides when it's +"up", optionally handing back a first value, and the parent stays +blocked until that moment: + +.. code:: python + + # trio, in-process + first = await nursery.start(child_fn) + + # tractor, cross-process + async with portal.open_context(child_fn) as (ctx, first): + ... + +The ``as (ctx, first)`` tuple is exactly that pair: the +:class:`tractor.Context` handle plus whatever value the child +passed to ``await ctx.started(value)``. And readiness is not +optional — for instance opening a stream before ``.started()`` +has been called raises a ``RuntimeError``; handshake first, then +dialog. + +Bidirectional streaming over a context +-------------------------------------- + +The canonical ping-pong (design history: `#53`_, `#223`_) — a +full-duplex msg stream between a parent and its spawned peer: + +.. literalinclude:: ../../examples/rpc_bidir_streaming.py + :caption: examples/rpc_bidir_streaming.py + :language: python + +What's going on? + +- ``start_actor()`` spawns the daemon-style subactor + ``'rpc_server'`` with this very module in its allowlist. + +- ``portal.open_context(simple_rpc, data=10)`` fires the + ``Start`` msg then blocks until the child task calls + ``await ctx.started(data + 1)`` — hence ``sent == 11``. + +- both tasks enter ``ctx.open_stream()``: a stream dialog is only + fully open once *each* side has entered its block. + +- the parent seeds the first ``'ping'``; each side then echoes + the other, one ``Yield`` msg per ``stream.send()``. + +- after the 9th pong the parent ``break``\ s (10 pings sent in + total) and exits its stream block, which sends ``Stop``; the + child's ``async for`` completes gracefully and its ``else`` + clause asserts all 10 pings arrived. + +- the 10th in-transit pong? Discarded by the implicit drain at + ``open_context()`` exit, which runs the dialog down to the + child's ``Return`` (here ``None``). + +- daemon actors live until told otherwise: + ``portal.cancel_actor()`` reaps the subactor explicitly. + +Results: the ``Return`` leg +--------------------------- + +Every context resolves to a final outcome. Wait on it explicitly +from the parent side: + +.. code:: python + + async with portal.open_context(ep) as (ctx, first): + ... + result = await ctx.wait_for_result() + +or just exit the block — ``__aexit__`` implicitly drains the msg +flow until the ``Return`` (or ``Error``) arrives, discarding any +in-transit ``Yield``\ s on the way. Either way the rule of +`causality`_ holds exactly as in a local nursery: **the opener +never unblocks before the remote task is done**. + +For post-hoc inspection (think supervision/restart logic) the ctx +also exposes ``Context.outcome``, ``.maybe_error`` and +``.has_outcome`` — where a "result" might well be the error the +dialog ended with. + +Cancellation semantics +---------------------- + +The part you actually came for; read it twice B) + +A context's two tasks are **cancel-scope-linked across the IPC +boundary**: whatever ends one side — error, cancellation, plain +old return — is relayed such that the other side ends +equivalently. No silent half-open dialogs, no orphaned remote +tasks, ever. + +``ctx.cancel()`` cancels the *remote* task +******************************************* + +:meth:`tractor.Context.cancel` requests cancellation of the +**remote** task only: + +.. code:: python + + async with portal.open_context(ep) as (ctx, first): + await accomplish_things(ctx) + await ctx.cancel() # remote task, NOT me + +A :class:`tractor.Context` is **not** a :class:`trio.CancelScope`: +the call doesn't (and can't) cancel your local task. It sends the +cancel request and waits a bounded ``timeout`` for the peer +runtime's ``CancelAck``, then your code proceeds to the block exit +as normal. + +Compare scopes here: ``Portal.cancel_actor()`` is the big hammer +which cancels the peer's **entire runtime** (and thus process); +``ctx.cancel()`` is the per-dialog scalpel. + +``ContextCancelled`` and the absorption rule +********************************************* + +When a context task gets cancelled *by request* the requestee's +runtime reports back with a :exc:`tractor.ContextCancelled` +("ctxc") whose ``.canceller`` field holds the uid of the actor +which asked. That one field decides what you observe: + +**you requested it** + i.e. ``ctxc.canceller == tractor.current_actor().uid``: the + ctxc is **absorbed** at ``open_context()`` exit — nothing + raises in your block. You asked for a graceful stop and got + it; if you care, ``await ctx.wait_for_result()`` hands the + ctxc back as a plain *value* for inspection. + +**anyone else requested it** + the peer cancelling itself, or some third actor cancelling it + from the side: the ctxc **is raised** in your block. From + your scope's perspective a task you depend on was killed out + from under you and SC demands you hear about it — exactly + like a sibling crash in a `nursery`_. + +In code: + +.. code:: python + + try: + async with portal.open_context(ep) as (ctx, first): + ... + except tractor.ContextCancelled as ctxc: + # can only be a peer- or third-party cancel; + # self-requested cancels are absorbed at exit. + assert ctxc.canceller != tractor.current_actor().uid + +This self- vs cross-cancel split is what makes explicit teardown +*composable*: a supervisor cancels its dialogs without try/except +noise, while unexpected cancellation anywhere in the tree still +propagates loudly like any other failure. + +.. warning:: + + Once ``ctx.cancel()`` has been called the dialog is done: a + subsequent ``ctx.open_stream()`` raises ``RuntimeError``. + +For introspection the ctx exposes trio-flavored status props: +``.cancel_called`` (this side requested), ``.cancel_acked`` (peer +confirmed), ``.cancelled_caught`` and ``.canceller`` — +deliberately mirroring :class:`trio.CancelScope` naming. + +Errors propagate, both ways +--------------------------- + +A crash on either end tears down the pair, SC style: + +- **child raises**: the exception ships back as an ``Error`` msg + and re-raises in the parent block boxed as a + :exc:`tractor.RemoteActorError`; the original class rides along + as ``.boxed_type`` with ``.src_uid`` naming the crashed actor. + +- **parent raises** (or is cancelled) inside the block: an + equivalent error/cancel is relayed to the child task so it can + never outlive the dialog. + +.. code:: python + + try: + async with portal.open_context(ep) as (ctx, first): + ... + except tractor.RemoteActorError as rae: + if rae.boxed_type is ValueError: + ... # remote ValueError, type preserved + +Errors that hop through intermediary actors on their way up the +tree ("inceptions" XD) keep the full relay trail in +``.relay_uid`` / ``.relay_path``. Payloads violating your declared +``pld_spec`` surface as the IPC analog of a ``TypeError``: +:exc:`tractor.MsgTypeError`. + +Overruns and backpressure +------------------------- + +Stream msgs land in a bounded per-context buffer on the receiver +side. A sender that outpaces a non-consuming receiver *overruns* +it and the runtime raises :exc:`tractor.StreamOverrun` (also a +:exc:`trio.TooSlowError`) instead of buffering without bound — SC +discipline applies to memory too. + +Your knobs: + +- ``msg_buffer_size`` on ``ctx.open_stream()`` sizes the buffer. + +- ``allow_overruns=True`` (on ``Portal.open_context()`` and/or + ``ctx.open_stream()``) opts in to absorbing overflow instead of + erroring — reasonable for bursty telemetry-ish feeds, just know + you're trading the error for extra buffering. + +One context, one stream +----------------------- + +A ``MsgStream`` is strictly **one-shot use**: once it closes — +gracefully or not, from either side — it can never be re-opened +on the same ctx. Want another round with the same peer? Open a +fresh context; they're cheap. The full close-vs-cancel teardown +story lives in :doc:`/guide/streaming`. + +.. rubric:: Where to next? + +:doc:`/guide/streaming` covers the rest of the msg-moving story: +the legacy one-way API, multi-actor pipelines and in-actor +broadcast fan-out. For exhaustive API detail see +:class:`tractor.Context`, :class:`tractor.MsgStream` and +:exc:`tractor.ContextCancelled`. + +.. _trio: https://github.com/python-trio/trio +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ +.. _nursery: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning +.. _causality: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#c-c-c-c-causality-breaker +.. _cheap or nasty: https://zguide.zeromq.org/docs/chapter7/#The-Cheap-or-Nasty-Pattern +.. _(un)protocol: https://zguide.zeromq.org/docs/chapter7/#Unprotocols +.. _msgspec: https://jcristharif.com/msgspec/ +.. _#53: https://github.com/goodboy/tractor/issues/53 +.. _#223: https://github.com/goodboy/tractor/issues/223 diff --git a/docs/guide/debugging.rst b/docs/guide/debugging.rst new file mode 100644 index 000000000..a4a7d3d47 --- /dev/null +++ b/docs/guide/debugging.rst @@ -0,0 +1,545 @@ +.. _debugging: + +================================ +"Native" multi-process debugging +================================ + +``tractor`` ships the thing every ``multiprocessing`` user has +wished for and quietly assumed was impossible: a multi-process +debugger that *just works*. + +Drop ``await tractor.pause()`` — or, with `greenback`_ installed, +a plain builtin ``breakpoint()`` — anywhere in any actor: the +root, a child, a grandchild, a sync helper function, even an +``asyncio`` task inside an "infected" actor. A full-featured +`pdbp`_ REPL opens *in that process*, with syntax-highlighted +source listings, tab completion and sticky mode, attached to your +one terminal. + +Under the hood every REPL entry acquires a tree-global tty mutex +via an IPC request to the root actor, so prompts from concurrent +pauses and crashes never interleave. ``ctrl-c`` is shielded while +any REPL is live, so a stray ``SIGINT`` can't vaporize the tree +out from under you. And in debug mode any uncaught error drops +you into a crash REPL *first in the failing child*, then again at +each parent as the boxed :class:`~tractor.RemoteActorError` climbs +the supervision tree. + +No remote-pdb sockets, no ``set_trace()`` port juggling, no +``ptrace`` attach dance: the debugger semantics you already know, +transparently extended across an entire process tree. Because +``tractor`` is a `structured concurrency`_ (SC) runtime, the +debugger composes with supervision instead of fighting it — quit +a REPL and errors keep propagating exactly like `trio`_ taught +you, ending in clean, zombie-free teardown. + +We're pretty sure it's the (first ever?) "native" debugging UX +for multi-process Python B) + +Enabling debug mode +------------------- + +Pass ``debug_mode=True`` to your runtime entrypoint, either +:func:`tractor.open_nursery` (which forwards it to the implicitly +opened root actor) or :func:`tractor.open_root_actor` directly: + +.. code:: python + + async with tractor.open_nursery( + debug_mode=True, # arm the whole actor tree + ) as an: + ... + +This arms the debug machinery *tree-wide*: + +- crash handling is enabled in every actor: uncaught errors enter + a REPL before they propagate, +- the internal tty-lock module is auto-exposed over RPC to every + subactor (this is what makes the one-terminal handoff work), +- console logging is bumped to include ``PDB``-level status msgs + so you can see REPL acquire/release events as they happen. + +You can instead flip it on for just one child, letting its +siblings crash-and-burn the normal way: + +.. code:: python + + portal = await an.start_actor( + 'sketchy_worker', + debug_mode=True, # OR-ed with the tree-wide flag + ) + +See ``examples/debugging/per_actor_debug.py`` for a runnable +proof of the selective style. + +.. note:: + + Debug mode requires the child-side runtime to be + ``trio``-native so that the tty-lock IPC dialog works; it's + currently supported on the ``'trio'`` (default) and + ``'main_thread_forkserver'`` spawn backends and raises + ``RuntimeError`` for any other ``start_method``. + +Your first pause point +---------------------- + +:func:`tractor.pause` is the SC-aware, multi-process spelling of +the stdlib's ``breakpoint()``. In the root actor it looks almost +boring: + +.. literalinclude:: ../../examples/debugging/root_actor_breakpoint.py + :caption: examples/debugging/root_actor_breakpoint.py + :language: python + +Run it and you get a ``(Pdb+)`` prompt parked on the ``pause()`` +line; type ``c`` (continue) and the program finishes normally. + +The exact same call works from *any* subactor, no matter how deep +in the tree: + +.. literalinclude:: ../../examples/debugging/subactor_breakpoint.py + :caption: examples/debugging/subactor_breakpoint.py + :language: python + +Each loop iteration the child actor requests the terminal from +the root over IPC, REPLs you, then releases it on ``c``. Pause +points are re-entrant-safe: repeat calls from the same task are +no-op'd and other local tasks queue politely for the REPL. + +When you get bored, type ``q`` (quit): the resulting +``bdb.BdbQuit`` is boxed and shipped to the parent like any other +remote error XD — causality is preserved even for your debugging +mistakes. + +Crash REPLs: errors climb the tree +---------------------------------- + +Pause points are only half the story. With debug mode armed, any +*uncaught* error anywhere in the tree triggers what we call crash +handling mode: + +.. literalinclude:: ../../examples/debugging/subactor_error.py + :caption: examples/debugging/subactor_error.py + :language: python + +What happens when the child hits that (very intentional) +``NameError``: + +1. a REPL opens **in the crashed child first** — you inspect the + raising frame, its locals, the works, right inside the failed + process, +2. when you quit, the error is boxed into a + :class:`~tractor.RemoteActorError` and relayed to the parent, +3. the parent (here the root) gets *its own* crash REPL with the + rendered remote traceback, +4. quit again and the nursery tears the tree down — errors keep + propagating per SC rules, no zombies left behind. + +You debug the failure at every hop of the supervision tree, which +for multi-hop trees means you can chase an error from the leaf +that raised it all the way up to the root that supervises it. + +Need to skip REPL entry for certain exceptions? Pass a predicate +via ``open_root_actor(debug_filter=...)``; by default +cancellation-only exception (groups) don't engage the REPL. + +One terminal, many actors +------------------------- + +So how do N processes share one tty without garbling it? The root +actor owns stdio for the whole tree and guards it with a FIFO +mutex; every subactor REPL entry is an IPC lock request to the +root. Exactly one actor-task in the entire tree can own the +terminal at a time, so prompts never interleave — ever. + +.. d2:: diagrams/debug_lock.d2 + :caption: Every REPL entry serializes through the root actor's + tty lock; ``continue``-ing one REPL hands the terminal to + the next waiter, FIFO style. + :alt: sequence diagram of two subactors serializing pdb REPL + access through the root actor's tty lock + +The runtime's teardown paths cooperate too: a cancelling parent +always waits for any live REPL to release before reaping +children, so the debugger never gets yanked out from under you +mid-keystroke. + +.. margin:: Watch the tree live + + Run any of these examples with a process-tree watcher in a + second terminal and watch actors come and go:: + + watch -n 0.1 "pstree -a $$" + +Here's the showpiece: one daemon child re-entering +``tractor.pause()`` forever inside a stream, while its sibling +repeatedly raises a ``NameError``: + +.. literalinclude:: ../../examples/debugging/multi_daemon_subactors.py + :caption: examples/debugging/multi_daemon_subactors.py + :language: python + +What you'll actually see +************************ + +Running it looks *roughly* like this (uids, tracebacks and source +listings elided; REPL order can vary with who wins the lock +race):: + + $ python examples/debugging/multi_daemon_subactors.py + + Opening a pdb REPL in paused actor: ('bp_forever', '') + + (Pdb+) c + + Opening a pdb REPL in crashed actor: ('name_error', '') + + (Pdb+) q + + Opening a pdb REPL in crashed actor: ('root', '') + + (Pdb+) q + +Two (then three) processes, one terminal, zero confusion: +``c``-ing out of the paused daemon's REPL releases the tty lock, +which immediately hands the prompt to the crashed sibling; quit +that and the error propagates as a fully-rendered +:class:`~tractor.RemoteActorError` to the parent where one final +crash REPL catches it before clean, zombie-free teardown. + +For maximum drama run +``multi_nested_subactors_error_up_through_nurseries.py`` (under +``examples/debugging/``) which pulls the same trick across a +*three-deep* process tree — the tty lock keeps every prompt +orderly the whole way up. + +Post-mortem, on demand +---------------------- + +Crash handling is automatic, but you can also enter a REPL on +a live exception *manually* with :func:`tractor.post_mortem` — +the actor-aware equivalent of ``pdb.post_mortem()`` — from inside +any ``except`` block in any actor (kwargs: ``tb=`` for an +explicit traceback, plus ``shield=`` and ``hide_tb=``): + +.. literalinclude:: ../../examples/debugging/pm_in_subactor.py + :caption: examples/debugging/pm_in_subactor.py + :language: python + +This example demos three REPL entries from one error: + +- the child's manual ``post_mortem()`` inside its ``except``, +- the runtime's automatic crash handler in the same child once + the error re-raises out of the RPC task, +- a manual ``post_mortem()`` in the parent on the received + :class:`~tractor.RemoteActorError`, whose ``.boxed_type`` + faithfully reports the original ``NameError``. + +Pausing from sync code +---------------------- + +No ``await``? No problem. :func:`tractor.pause_from_sync` brings +the same tree-aware REPL to plain synchronous functions — handy +when the suspect code is three helpers deep and decidedly not +async. + +It's powered by `greenback`_, which is optional, so you need to: + +1. install it (it ships in ``tractor``'s ``sync_pause`` + dependency group), +2. enable it at runtime entry: + +.. code:: python + + async with tractor.open_nursery( + debug_mode=True, + maybe_enable_greenback=True, + ) as an: + ... + +With that armed, sync code can pause from three different caller +environments: the main ``trio`` thread, ``trio.to_thread`` bg +threads, and (see the next section) ``asyncio`` tasks in infected +actors. The greenback "portal" hops back into the ``trio`` loop +to do the lock/REPL dance on your behalf: + +.. literalinclude:: ../../examples/debugging/sync_bp.py + :caption: examples/debugging/sync_bp.py (the sync fn, excerpt) + :language: python + :pyobject: sync_pause + +.. literalinclude:: ../../examples/debugging/sync_bp.py + :caption: examples/debugging/sync_bp.py (called in a subactor, + excerpt) + :language: python + :pyobject: start_n_sync_pause + +The full script also exercises the hairier root-actor bg-thread +cases (and documents their remaining sharp edges) if you want the +deep lore. + +The builtin ``breakpoint()`` override +************************************* + +When debug mode boots with greenback available, ``tractor`` wires +Python's `PEP 553`_ hook so the *builtin* ``breakpoint()`` becomes +the actor-aware sync pause, by exporting:: + + PYTHONBREAKPOINT=tractor.devx.debug._sync_pause_from_builtin + +That means third-party and legacy code containing bare +``breakpoint()`` calls debugs correctly inside your actor tree +with zero edits (the override even forwards kwargs like +``hide_tb`` to the underlying pause machinery, as shown in the +excerpt above). + +.. warning:: + + Without greenback (or with ``maybe_enable_greenback=False``, + the default), ``debug_mode=True`` instead *blocks* the builtin + ``breakpoint()``: ``sys.breakpointhook`` is swapped for a + raiser and ``PYTHONBREAKPOINT=0`` is set. A naive + ``breakpoint()`` from some random process would clobber the + shared tty, so we'd rather hand you a loud ``RuntimeError`` + with install instructions. + +Both the hook and the env var are restored to their prior values +on runtime exit — see +``examples/debugging/restore_builtin_breakpoint.py`` for the +proof. + +Breakpoints inside ``asyncio`` tasks +------------------------------------ + +Yes, even "infected ``asyncio``" actors get the goods. Spawn a +child with ``infect_asyncio=True`` (``trio`` runs as a guest on +the ``asyncio`` loop inside it) and, with debug mode + greenback +armed, every ``asyncio`` task started via ``tractor.to_asyncio`` +is automatically granted a greenback portal — so a plain builtin +``breakpoint()`` (or ``tractor.pause_from_sync()``) inside an +``asyncio.Task`` joins the same single-terminal, tree-locked REPL +flow: + +.. literalinclude:: ../../examples/debugging/asyncio_bp.py + :caption: examples/debugging/asyncio_bp.py + :language: python + +Note the interleave: a ``breakpoint()`` on the ``asyncio`` side, +``tractor.pause()`` on the ``trio`` side of the same actor, and +another pause up in the root — all serialized through the one tty +lock with no cross-actor (or cross-event-loop!) clobbering. + +One catch: ``asyncio`` tasks spawned *out-of-band* — i.e. not via +``tractor.to_asyncio``, typically by some third-party aio lib — +have no portal bestowed, so a sync pause from one raises a loud +``RuntimeError`` telling you to ``greenback.ensure_portal()`` +first. See :ref:`the caveats ` below. + +Teardown debugging: the shielded pause +-------------------------------------- + +`Cancellation`_ is ``trio``'s bread and butter, which raises an +awkward question: how do you REPL inside an *already-cancelled* +scope, say while debugging some teardown sequence? A bare +``pause()`` would itself be cancelled at its next checkpoint. + +The answer is ``await tractor.pause(shield=True)``, which wraps +the lock acquisition and REPL session in a shielded cancel scope +(``post_mortem(shield=True)`` works the same way): + +.. literalinclude:: ../../examples/debugging/shielded_pause.py + :caption: examples/debugging/shielded_pause.py + :language: python + +If you forget, ``tractor`` has your back: an unshielded +``pause()`` from a cancelled scope fails fast with a hint +suggesting ``await tractor.pause(shield=True)`` instead of +silently never REPL-ing. + +Go ahead, mash ctrl-c +--------------------- + +While any REPL is live the runtime installs a custom ``SIGINT`` +handler tree-wide so that a reflexive ``ctrl-c`` (or five) can't +nuke your debug session: + +- the actor that owns the REPL ignores the interrupt and simply + re-flushes the prompt — keep mashing, it's fine, +- the root actor ignores ``SIGINT`` while a still-IPC-connected + child holds the tty lock, so the supervisor won't tear down the + tree out from under the debugger, +- if the lock state has gone *stale* — the locking child died or + its IPC channel dropped — the root cancels the stale lock scope + and restores ``trio``'s default handler, so ``ctrl-c`` works + again exactly when it should. + +The handler is uninstalled and ``trio``'s own ``SIGINT`` +semantics restored every time a REPL releases (on ``continue`` / +``quit``). + +Live task-tree dumps +-------------------- + +Sometimes there's no error to catch — the tree is just *hung* and +you want to know where. For that ``tractor`` integrates +`stackscope`_: send a signal, get a full ``trio`` task-tree dump +from every actor in the tree. + +Enable it any of three ways: + +- ``open_root_actor(enable_stack_on_sig=True)`` (or via + ``open_nursery()`` which forwards it), +- set ``TRACTOR_ENABLE_STACKSCOPE=1`` in the env — it's inherited + through the process tree so every (sub)actor arms the handler + at boot, +- call ``tractor.devx.enable_stack_on_sig()`` directly. + +It's intentionally *not* gated on ``debug_mode`` so you can leave +it armed in plain runs. Then, when the hang strikes, signal the +tree with ``SIGUSR1``. + +.. tip:: + + No need to hunt down pids — pattern-match the original cmdline + with ``pkill``:: + + $ pkill --signal SIGUSR1 -f "python example_script.py" + +Each actor dumps its entire ``trio`` task tree (full nursery +recursion via ``stackscope.extract()``) to its tty *and* tees it +to ``/tmp/tractor-stackscope-.log`` — so the trace survives +even under captured-stdio harnesses — then relays the signal on +to its children, parent-before-child, until the whole tree has +reported in. + +Try it yourself with the demo script, which deliberately hangs a +subactor in a shielded sleep: + +.. literalinclude:: ../../examples/debugging/shield_hang_in_sub.py + :caption: examples/debugging/shield_hang_in_sub.py + :language: python + +(That ``trio.CancelScope(shield=True)`` hang also shows off the +zombie reaper: ``ctrl-c`` the root and the un-cancellable child +still gets hard-reaped — if you can create a zombie it **is a +bug**.) + +Crash handling for sync and CLI code +------------------------------------ + +All of the above rides on the actor runtime, but crashes don't +politely wait for ``trio.run()``. For plain sync code — think +``typer``/``click`` CLI endpoints, config parsing, anything +pre-runtime — there's a sync context manager that wraps the same +``pdbp`` post-mortem UX: + +.. code:: python + + from tractor.devx import open_crash_handler + + def main(): # any sync code, no runtime required + with open_crash_handler() as boxed: + run_my_cli_thing() + +By default any ``BaseException`` (minus an ``ignore`` set +defaulting to ``KeyboardInterrupt`` and ``trio.Cancelled``) +enters the REPL then re-raises on exit; pass +``raise_on_exit=False`` to suppress instead and introspect the +``boxed.value`` afterward. The ``catch``/``ignore`` sets and a +``repl_fixture`` are all tweakable. + +For the classic ``--pdb`` CLI-flag pattern use the conditional +variant: + +.. code:: python + + from tractor.devx import maybe_open_crash_handler + + @app.command() # a `typer` (or `click`) endpoint + def cmd(pdb: bool = False): + with maybe_open_crash_handler(pdb=pdb): + ... + +REPL niceties and hooks +----------------------- + +Every REPL in this guide is a `pdbp`_ instance (the maintained +fork-and-fix of `pdb++`_) pre-configured by ``tractor``: + +- pygments syntax highlighting in listings and tracebacks, +- tab completion — including an automatic fixup for + libedit-compiled CPythons (e.g. ``uv``-distributed pythons), +- sticky mode available via the ``sticky`` command (off by + default), +- no long-line truncation (terminal resizes behave), +- the ``(Pdb+)`` prompt, ``ll``, hidden-frames support and the + rest of the ``pdb++`` goodies you may already know. + +Internal runtime frames are traceback-hidden so the REPL lands +exactly on *your* ``pause()``-call or crash frame, never on +``tractor`` guts. + +Finally, if your app owns the terminal (TUIs, fullscreen +dashboards) pass ``repl_fixture=`` to ``pause()``, +``post_mortem()`` or ``open_crash_handler()``: it's entered just +before the REPL engages (return ``False`` to skip entry entirely) +and exited on release — perfect for suspending and restoring your +screen around a debug session. + +.. _debugging-caveats: + +Caveats and platform notes +-------------------------- + +An honest list of the current rough edges: + +- **Windows**: the debugger has no CI coverage on windows at all + (the entire test module is skipped there); manual testing has + shown it *can* work, but you're in uncharted territory — + reports welcome! +- **macOS**: supported but with rough edges: special-cased prompt + re-flushing for ``bash``-on-darwin, a few tooling tests skipped + on CI, and the AF_UNIX ~104-char socket-path limit forces some + examples (like the stackscope demo above) to fall back from + ``'uds'`` to ``'tcp'`` transport. Wonder if all of it'll work + on OS X? So do we. +- **CPython 3.14**: ``greenback`` (via ``greenlet``) doesn't + support 3.14 yet, so ``pause_from_sync()`` and the builtin + ``breakpoint()`` override are effectively 3.13-only for now. + The async APIs — ``pause()`` and ``post_mortem()`` — need no + greenback and work everywhere. +- **out-of-band** ``asyncio`` **tasks**: sync pauses from aio + tasks *not* spawned via ``tractor.to_asyncio`` raise a + ``RuntimeError`` (no greenback portal was bestowed); run + ``await greenback.ensure_portal()`` inside such a task first. +- **nested-tree ctrl-c edges**: ``SIGINT`` relay through + intermediary parents that aren't themselves in debug mode still + has known rough edges — see `#320`_. +- **captured stdio**: ``pytest``-style output capture can hang a + ``pause()``; use a real terminal (or a pty à la ``pexpect``, + which is how ``tractor``'s own suite drives every one of these + examples). + +Where to next? +-------------- + +.. seealso:: + + - :doc:`/guide/context` — the SC-linked cross-actor task API + that all the crash-propagation semantics above ride on. + - :func:`tractor.pause`, :func:`tractor.post_mortem` and + :func:`tractor.pause_from_sync` in the API reference. + - ``examples/debugging/`` — 20-odd runnable scripts, nearly + every one exercised by the test suite through a real pty. + +.. _structured concurrency: + https://en.wikipedia.org/wiki/Structured_concurrency +.. _trio: https://github.com/python-trio/trio +.. _cancellation: https://trio.readthedocs.io/en/latest/ + reference-core.html#cancellation-and-timeouts +.. _pdbp: https://github.com/mdmintz/pdbp +.. _pdb++: https://github.com/pdbpp/pdbpp +.. _greenback: https://github.com/oremanj/greenback +.. _stackscope: https://github.com/oremanj/stackscope +.. _PEP 553: https://peps.python.org/pep-0553/ +.. _#320: https://github.com/goodboy/tractor/issues/320 diff --git a/docs/guide/discovery.rst b/docs/guide/discovery.rst new file mode 100644 index 000000000..cc76a7f3b --- /dev/null +++ b/docs/guide/discovery.rst @@ -0,0 +1,255 @@ +Actor discovery +=============== + +So you've spawned a tree of trio-"actors"; now their tasks need to +*find* each other to start a dialog. ``tractor`` ships a (self +admittedly) **very naive** discovery system which is nonetheless +mighty handy for wiring up service-style apps: a built-in +*registrar* actor plus a small set of lookup APIs that deliver +a live, connected ``Portal`` to whichever peer you're after. + +.. d2:: diagrams/actor_tree.d2 + :margin: + :caption: The root actor doubles as the *registrar* by + default; every spawned actor registers itself with it. + :alt: actor tree with root acting as registrar + +Because ``tractor`` is built on structured concurrency (SC), the +discovery layer is *not* some external etcd/consul-shaped service +you have to babysit; it's just another actor — normally the root +of your tree — doing a bit of bookkeeping as part of the runtime. + +Every actor phones home +----------------------- + +On runtime boot **every** actor self-registers with the registrar: +it submits its unique ``(name, uuid)`` identity pair (aka its +``uid``) mapped to the list of transport addresses its IPC server +is bound to. On graceful teardown it likewise *un*-registers, so +the registry tracks the live tree as it grows and shrinks. + +.. note:: + Actor names are **not** enforced unique — the registry is keyed + by the full ``(name, uuid)`` pair. Name-based lookups simply + resolve to the *last* registered match, so if you boot five + actors all named ``'bob'``, you get the freshest ``'bob'`` B) + +First boot: who's the registrar? +-------------------------------- + +By default the **root actor** *is* the registrar; subactors +inherit the tree's ``registry_addrs`` at spawn time so the whole +clan shares one registry with zero config on your part. + +The bootstrap rule inside ``open_root_actor()`` is delightfully +simple: + +- on boot, ping every socket addr in ``registry_addrs``; when none + are passed the per-transport defaults are used: for TCP the + loopback ``('127.0.0.1', 1616)``, for UDS a + ``registry@1616.sock`` file, + +- if a registrar answers, you boot as a plain (non-registrar) root + actor and register with the *existing* registry; your own IPC + server binds random same-transport addrs instead, + +- if **nothing answers, congratulations: you just became the + registrar**. Your transport server binds the registry addrs + themselves and you start serving lookups for everyone else. + +Pass ``ensure_registry=True`` when your program *requires* being +the one-and-only registrar; boot then fails loudly with a +``RuntimeError`` if some other process already bound the registry +socket(s). + +Looking up actors +----------------- + +All lookup APIs are async context managers, so the SC rule you +already know from the rest of ``tractor`` holds here too: any +delivered portal (and its underlying IPC channel) is scoped to +your ``async with`` block — no dangling connections. + +``find_actor()`` +**************** + +The workhorse: ask the registrar for ``name`` and connect a portal +to the match, or get ``None`` back when nobody's home: + +.. code:: python + + async with tractor.find_actor('data_feed') as portal: + if portal is None: + ... # not registered anywhere; maybe spawn it? + else: + await portal.run(do_stuff) + +Knobs worth knowing: + +- ``registry_addrs=[...]``: query specific (possibly multiple, + possibly remote) registrars instead of your tree's default, + +- ``only_first=False``: deliver a ``list[Portal]`` of *all* + matches found across the queried registrars instead of just the + first, + +- ``raise_on_none=True``: raise a ``RuntimeError`` instead of + yielding ``None`` when no match is found — for when absence is + a hard error in your app. + +``wait_for_actor()`` +******************** + +Blocks until *someone* registers under ``name``, then yields a +portal to that registree. Perfect for "wait for my sibling service +to come up" sequencing: + +.. code:: python + + async with tractor.wait_for_actor('service') as portal: + await portal.run(some_fn) + +``query_actor()`` +***************** + +A lookup *without* connecting to the target: yields an +``(addr, reg_portal)`` pair where ``addr`` is the peer's preferred +transport address, or ``None`` when nothing is registered under +that name. Use it for liveness peeks or to log where a service +lives without actually dialing it up. + +``get_registry()`` +****************** + +Yields a portal straight to the registrar actor itself — or a +``LocalPortal`` shim when the calling actor *is* the registrar +(no IPC required to talk to yourself, hopefully). + +Fast paths and address preference +--------------------------------- + +Before doing any RPC to the registrar, every lookup first scans +the calling actor's *already-connected peers*: if you have a live +channel to an actor named ``name`` you get a portal over it +immediately, no registrar round-trip at all. + +When a registry entry holds *multiple* addresses (a multihomed +actor) the "best" one is chosen by locality: + +1. UDS — same-host guaranteed, lowest overhead, + +2. local TCP — loopback or any of this host's own interface + addrs, + +3. remote TCP — the only option when actually distributed. + +Within a tier the most recently registered addr wins. Stale +entries (an addr that no longer accepts connections) are detected +on use and deleted from the registrar's table on your behalf. + +Demo: register and find a service +--------------------------------- + +The simplest possible spin: start a subactor, ask the registrar +where it lives, and wait on its registration: + +.. literalinclude:: ../../examples/service_discovery.py + :caption: examples/service_discovery.py + :language: python + +The daemon-service pattern +-------------------------- + +The classic deployment shape: a long-lived daemon actor serves +RPC, later-running code discovers it by name, calls in, and +gracefully cancels it when the job is done: + +.. literalinclude:: ../../examples/service_daemon_discovery.py + :caption: examples/service_daemon_discovery.py + :language: python + +Note the teardown ordering — *graceful cancel* of the daemon via +its portal is part of the pattern; under SC a "service" is still +somebody's child and somebody is responsible for reaping it. + +Joining an existing tree from outside +------------------------------------- + +Discovery isn't limited to a single program: any standalone script +can join a running tree by booting its *own* root actor pointed at +the existing registrar: + +.. code:: python + + import trio + import tractor + + async def main(): + async with ( + # contact the live tree's registrar + tractor.open_root_actor( + registry_addrs=[('127.0.0.1', 1616)], + ), + tractor.find_actor('data_feed') as portal, + ): + ... # RPC away like you were born here + + trio.run(main) + +Per the bootstrap rules above, if the registrar at those addrs is +*not* reachable this process simply becomes its own (registrar) +root — so the same code works standalone and as a tree-joiner. + +"Arbiter"? A legacy naming note +------------------------------- + +In older releases (and many an old blog post or issue thread) the +registrar actor was called the *arbiter*, with matching APIs like +``get_arbiter()`` and an ``arbiter_addr`` argument. All of that +terminology is retired: it's *registrar*/*registry* everywhere now +(``registry_addrs``, ``get_registry()``, ...) and the +``tractor.Arbiter`` export survives only as a back-compat alias of +``tractor.Registrar``. If you see "arbiter" somewhere, mentally +substitute "registrar" and you're up to date. + +.. note:: + Multihoming nerds: ``tractor.discovery`` also ships + libp2p-style *multiaddr* helpers — ``mk_maddr()`` and + ``parse_maddr()`` — for describing transport endpoints as + structured strings. + +Very naive, very honest +----------------------- + +To be clear, this is a **very naive** discovery system: one +process-tree-local registrar holding a dict, no replication, no +re-election when it dies, no cross-host propagation. That's +intentional (for now); it covers the "wire up my services on this +host" case without dragging in a consensus protocol. + +On the roadmap (issue `#216`_ tracks a chunk of it): + +- registrar high(er)-availability: staying up past tree teardown + and re-election, + +- a `gossip protocol`_ for decentralized cross-host discovery (the + zguide's `discovery`_ chapter is the spiritual reference), + +- `modern protocol`_ (rendezvous) style meet-up points. + +If any of that scratches your itch, the issue tracker would love +to hear from you. + +.. seealso:: + - :doc:`/guide/testing` — watching live actor trees (and their + registrar) while the test suite or your app runs. + - API refs: :func:`tractor.find_actor`, + :func:`tractor.wait_for_actor`, + :func:`tractor.query_actor`, + :func:`tractor.get_registry`, + :class:`tractor.Registrar`. + +.. _gossip protocol: https://en.wikipedia.org/wiki/Gossip_protocol +.. _modern protocol: https://en.wikipedia.org/wiki/Rendezvous_protocol +.. _discovery: https://zguide.zeromq.org/docs/chapter8/#Discovery +.. _#216: https://github.com/goodboy/tractor/issues/216 diff --git a/docs/guide/index.rst b/docs/guide/index.rst new file mode 100644 index 000000000..cbd4fe7b9 --- /dev/null +++ b/docs/guide/index.rst @@ -0,0 +1,52 @@ +Guides +====== +Task-focused walkthroughs of every major ``tractor`` +subsystem, each built around real, *test-suite +verified* example scripts from the repo's +``examples/`` dir (we never copy-paste code into +docs; what you read is what CI runs). + +Roughly in "first date to long term relationship" +order, + +- :doc:`spawning` — actor nurseries, daemons + + one-shot workers, process lifetimes. +- :doc:`rpc` — portals: calling into another + process like it's a local ``await``. +- :doc:`context` — the cross-actor task-pair + primitive at the heart of modern ``tractor``. +- :doc:`streaming` — one-way and bidirectional + msg streams between actors. +- :doc:`cancellation` — supervision, error + boxing + propagation, teardown discipline. +- :doc:`debugging` — the multi-process native + REPL debugger; our flagship DX feature B) +- :doc:`discovery` — the registrar, finding + actors by name, service patterns. +- :doc:`clustering` — quick flat process + clusters via one ``async with``. +- :doc:`parallelism` — worker pools without + pools; a ``concurrent.futures`` re-think. +- :doc:`asyncio` — "infected asyncio" mode: + SC-supervise ``asyncio`` tasks from ``trio``. +- :doc:`msging` — typed IPC payloads, the wire + msg-spec and custom codecs. +- :doc:`testing` — running + monitoring the + test suite (and testing your own actor apps). + +.. toctree:: + :hidden: + :maxdepth: 1 + + spawning + rpc + context + streaming + cancellation + debugging + discovery + clustering + parallelism + asyncio + msging + testing diff --git a/docs/guide/msging.rst b/docs/guide/msging.rst new file mode 100644 index 000000000..5c5a4d125 --- /dev/null +++ b/docs/guide/msging.rst @@ -0,0 +1,260 @@ +Typed messaging +=============== +Every value that crosses an actor boundary rides inside a typed +msg. ``tractor`` ships a small, fixed family of msg types, the +"SC-transitive supervision protocol", which encapsulates *all* +RPC dialogs in the tree such that `structured concurrency`_ (SC) +semantics -- parent-child task linkage, error propagation, +graceful cancellation -- hold across every process hop. On top of +that +protocol you can layer **your own** payload type contracts, +per-endpoint, and have them enforced at runtime by the codec. + +.. note:: + + Older posts and readmes claim ``tractor`` "uses + ``msgpack``(-python)" on the wire. The wire *encoding* is still + msgpack, but since ``0.1.0a5`` all codec work is done by + msgspec_ against a strictly-typed, tagged-union msg-spec; + neither ``msgpack-python`` nor ``u-msgpack`` are involved. + +The wire format +--------------- +Each protocol msg is a :class:`msgspec.Struct` subtype declared +with ``tag=True, tag_field='msg_type'``, so the full set decodes +as a `tagged union`__ with zero dispatch code of our own. The +payload-carrying msgs all inherit from ``PayloadMsg`` which boxes: + +__ https://jcristharif.com/msgspec/structs.html#tagged-unions + +- ``.cid`` -- the "context id" identifying which dialog (i.e. + which ``Context``) the msg belongs to, +- ``.pld`` -- the *payload*, aka your app's actual data. + +Decoding is deliberately two-layered: + +- the **transport codec** decodes only the protocol *envelope*, + intentionally leaving ``.pld`` as raw bytes + (:class:`msgspec.Raw`), +- a **per-context payload-receiver** (the internal ``PldRx``) then + decodes each ``.pld`` against *that* dialog's user-defined type + spec. + +This split is what lets every ``Context`` carry its own msg-spec +without reconfiguring the shared transport, keeps the runtime's +own traffic immune to your app's spec choices, and makes any +validation failure attributable to exactly one dialog (and thus +one task pair) instead of nuking the whole channel. + +The protocol family +------------------- +The entire msg-spec is ten types, all importable from +``tractor.msg`` (defined in ``tractor.msg.types``): + +.. list-table:: + :header-rows: 1 + :widths: 18 82 + + * - msg type + - role + * - ``Aid`` + - actor-identity handshake; the first thing two peers + exchange on connect (name, uuid, pid). + * - ``SpawnSpec`` + - parent -> child runtime config sent right after ``Aid``: + enabled modules, registry/bind addrs, runtime vars. + * - ``Start`` + - request to remotely schedule an RPC task: target + namespace + func name, kwargs and the caller's uid. + * - ``StartAck`` + - the callee's ack declaring the endpoint's "functype": + ``asyncfunc``, ``asyncgen`` or ``context``. + * - ``Started`` + - the first value passed to ``ctx.started()``; completes + the context handshake. + * - ``Yield`` + - one streamed value per ``MsgStream.send()`` call. + * - ``Stop`` + - graceful stream termination; the IPC rendition of + ``StopAsyncIteration``. + * - ``Return`` + - the final return value of the remote task fn. + * - ``CancelAck`` + - ``bool`` result of a runtime cancel-request; always + decodable so graceful cancellation can never be broken + by a custom msg-spec. + * - ``Error`` + - a boxed remote exception (src uid, relay path, tb str, + ..) relayed for local re-raise as + :class:`tractor.RemoteActorError`. + +Squint and you'll see an SC task scope serialized onto the wire: +every dialog opens with ``Start``/``StartAck`` (plus ``Started`` +for ``@tractor.context`` endpoints), optionally streams +``Yield``-s until a ``Stop``, and **always** terminates with +exactly one of ``Return``, ``Error`` or ``CancelAck``. That 1:1 +mapping of msg sequence onto a cross-process task pair is why we +call the protocol *SC-transitive*: supervision semantics survive +every hop of the tree. In `(un)protocol`_ terms it's our "SC +dialog un-protocol". + +For introspection the union alias ``tractor.msg.MsgType``, the +list ``__msg_types__`` and the spec alias ``__msg_spec__`` are +all exported. + +Payload typing with ``pld_spec`` +-------------------------------- +By default ``.pld`` may be any msgspec-supported type, i.e. the +spec is ``Any``. To constrain a single endpoint's dialog, pass +a type (union) to the decorator: +``@tractor.context(pld_spec=MyStruct|None)``. The spec then +applies to all payload-carrying msgs of that dialog -- +``Started``, ``Yield`` and ``Return`` -- on both sides of the +IPC. Pro tip: keep ``None`` in your union since most endpoints +implicitly ``return None`` and a bare ``ctx.started()`` ships +``None`` too. + +.. literalinclude:: ../../examples/typed_payloads.py + :caption: examples/typed_payloads.py + :language: python + +What's going on? + +- the payload schema is just a :class:`msgspec.Struct` subtype; + anything msgspec can tag and decode works, including unions + of structs, builtins and containers. +- decorating with ``@tractor.context(pld_spec=...)`` attaches the + spec to the endpoint; both peers' payload-receivers now decode + this dialog's payloads against it. No spec sharing files, no + IDL compiler, the contract *is* the Python type. +- the happy path looks identical to untyped code: the child calls + ``await ctx.started()``, streams or returns + more conforming values, and the parent receives fully decoded + struct instances (not dicts!) on its side. +- the sad path is the point: shipping a value *outside* the spec + raises :class:`tractor.MsgTypeError`, which the example catches + to show off the failure mode; see the anatomy section below for + exactly where it gets raised. + +Where validation happens: cheap-or-nasty +---------------------------------------- +A naive impl would validate every payload on both send *and* +receive, doubling your codec bill exactly where throughput +matters most. Instead ``tractor`` follows the 0mq lords' +"`cheap or nasty`_" pattern: be **nasty** (strict, eager, +expensive) on the rare control msgs and **cheap** (lazy, fast) on +the high-rate stream path. + +- ``Started`` is the *only* payload that gets the full nasty + treatment: ``ctx.started(value)`` stringently + **roundtrip-checks** the encoded msg against the dialog's spec + *before* sending, so a non-conforming first value raises + :class:`tractor.MsgTypeError` immediately in the child and + never even hits the wire. (You can opt out per-call with + ``ctx.started(..., validate_pld_spec=False)`` if you measure + a real cost.) +- ``Yield`` payloads are **never** checked inside + ``MsgStream.send()``; they're validated receiver-side on each + ``MsgStream.receive()``. A violation raises a ``MsgTypeError`` + in the receiver *and* relays an ``Error`` msg back so the + offending sender gets one raised too. +- the remaining control msgs (``Start``, ``Return``) are likewise + validated such that violations raise in the **sending** actor, + pointing the traceback at the code that actually goofed. + +Anatomy of a ``MsgTypeError`` +----------------------------- +:class:`tractor.MsgTypeError` is the IPC equivalent of a builtin +``TypeError``: a ``RemoteActorError`` subtype raised whenever +a msg fails to decode against the active spec. The useful bits: + +- ``.bad_msg`` -- the offending msg instance (reconstructed from + its wire form when necessary) so you can inspect the actual + ``.pld`` that broke the contract. +- ``.expected_msg_type`` -- the protocol msg type the bad msg was + (supposed to be) decoded as, e.g. ``Started[Point]``. +- plus the standard ``RemoteActorError`` goodies: ``.boxed_type``, + ``.src_uid``, ``.ipc_msg`` and the fancy ``.pformat()`` tb-box + rendering. + +Practical reading guide: a *sender-side* MTE (``Started``, +``Return``) points straight at your offending ``await +ctx.started()`` or ``return`` statement, while a *receiver-side* +MTE (``Yield``) surfaces from the consumer's ``receive()`` call +with the relay copy delivered back to the producer. Either way +the failure is scoped to that one dialog; sibling contexts on the +same channel keep right on trucking. + +Custom wire types: ``mk_codec()`` and friends +--------------------------------------------- +msgspec covers a wide set of `builtin types`__ natively; for +anything else you teach the codec via extension hooks. The +easiest path is per-endpoint: ``@tractor.context()`` accepts +``enc_hook``/``dec_hook`` params right alongside ``pld_spec``. +For full control build and apply a codec yourself; encode-side: + +__ https://jcristharif.com/msgspec/supported-types.html + +.. code:: python + + from tractor.msg import mk_codec, apply_codec + + codec = mk_codec( + enc_hook=nsp_to_str, # your-type -> wire-type + ext_types=[NamespacePath], + ) + with apply_codec(codec): # ContextVar-scoped override + ... # msgs sent by this task now encode NSPs + +and decode-side, scoped to an open context (note the import from +``tractor.msg._ops``, not yet re-exported): + +.. code:: python + + from tractor.msg._ops import limit_plds + + with limit_plds( + NamespacePath, + dec_hook=str_to_nsp, # wire-type -> your-type + ext_types=[NamespacePath], + ): + ... # this dialog's payloads decode as NSPs + +``apply_codec()`` is ``ContextVar``-scoped: it overrides the +codec for the current task (and only that task), not the whole +process. For complete working flows, including hook pairing rules +and roundtrip cases, see ``tests/msg/test_ext_types_msgspec.py`` +and ``tests/msg/test_pldrx_limiting.py``. + +The runtime dogfoods this pattern with +:class:`tractor.msg.NamespacePath`: a ``str``-subtype shaped like +``'module.path:obj_name'`` used for every RPC target reference. +It ships over the wire as a plain string yet ``.load_ref()``-s +back to the actual object in the receiving actor's memory domain; +a minimal "pointer type" for shared-nothing systems. + +Toward capability-based msging +------------------------------ +The ``pld_spec`` + codec-hook layer is the foundation for the +long-game: **capability-based msging** where each dialog's +type contract doubles as a capability grant, negotiated as part +of the protocol itself. That work is tracked in `#196`_ (with the +original typed-proto epic in `#36`_); if strongly-typed +distributed systems get you going, we'd love your input. + +Where to next? +-------------- +.. seealso:: + + - :doc:`/guide/context` for the dialog API these msgs + implement: ``started()``, streams and results. + - :doc:`/guide/asyncio` for shuttling (typed) payloads into + ``asyncio``-land via an infected subactor. + - the msgspec_ docs for everything your payload types can be. + +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _msgspec: https://jcristharif.com/msgspec/ +.. _cheap or nasty: https://zguide.zeromq.org/docs/chapter7/#The-Cheap-or-Nasty-Pattern +.. _(un)protocol: https://zguide.zeromq.org/docs/chapter7/#Unprotocols +.. _#196: https://github.com/goodboy/tractor/issues/196 +.. _#36: https://github.com/goodboy/tractor/issues/36 diff --git a/docs/guide/parallelism.rst b/docs/guide/parallelism.rst new file mode 100644 index 000000000..7a6dc1c23 --- /dev/null +++ b/docs/guide/parallelism.rst @@ -0,0 +1,176 @@ +Parallelism and worker pools +============================ + +The initial ask is almost always the same: *"how do i make a worker +pool?"* — i.e. the thing :mod:`multiprocessing` and +:class:`concurrent.futures.ProcessPoolExecutor` get reached for +once the GIL becomes the enemy. + +Here's the structured concurrency (SC) answer: ``tractor`` is built +to handle any SC process tree you can imagine; a "worker pool" +pattern is a trivial special case. So instead of shipping a pool +*class* with knobs bolted on, you compose one from the same two +ingredients used everywhere else in ``tractor``: an actor nursery +and some IPC. + +The stdlib baseline +------------------- + +For a fair comparison, start from the canonical +:class:`~concurrent.futures.ProcessPoolExecutor` primes example +straight out of the Python docs, + +.. literalinclude:: ../../examples/parallelism/concurrent_futures_primes.py + :caption: examples/parallelism/concurrent_futures_primes.py + :language: python + +Synchronous code, a hidden thread + IPC machine under the hood, and +an API surface (executors, futures, ``.map()``) invented to paper +over the fact that the pool isn't part of your program's task tree. +Keep an eye on three things for the rewrite: how work is submitted, +how results come back, and what happens when a worker dies. + +The ``tractor`` way +------------------- + +Now the same workload as a ``tractor`` program, + +.. literalinclude:: ../../examples/parallelism/concurrent_actors_primes.py + :caption: examples/parallelism/concurrent_actors_primes.py + :language: python + +What's different (and what isn't), + +- ``worker_pool()`` is ~30 lines of *your* code: an actor nursery + spawning ``workers`` subactors — each a full process running its + own ``trio`` task tree — kept alive and ready for work until the + block exits; ``enable_modules=[__name__]`` is the capability + allowlist letting them run this module's functions, +- jobs are "submitted" by just... calling the function: + ``portal.run(is_prime, n=value)`` runs ``is_prime()`` in a + worker and hands back its result like any local ``await``, +- results stream back through a plain + :func:`trio.open_memory_channel` *as they complete* — no futures + and no polling, +- teardown is one ``await tn.cancel()`` + (:meth:`tractor.ActorNursery.cancel`), and any worker crash + triggers the one-cancels-all machinery from + :doc:`/guide/cancellation` — a dead worker can never strand the + pool. + +This uses no extra threads, fancy semaphores or futures; all we +need is ``tractor``'s IPC! The full scorecard, + +.. list-table:: + :header-rows: 1 + :widths: 50 50 + + * - ``concurrent.futures`` + - ``tractor`` + * - ``ProcessPoolExecutor()`` + - ``worker_pool()`` — yours, ~30 lines + * - ``executor.map(is_prime, PRIMES)`` + - ``actor_map(is_prime, PRIMES)`` async-gen + * - ``Future`` + internal result queue + - :func:`trio.open_memory_channel` + * - results in input order + - results as they complete + * - worker crash -> ``BrokenProcessPool`` + - boxed :class:`tractor.RemoteActorError` + * - pool teardown on ``with`` exit + - one-cancels-all nursery teardown + +.. margin:: How many workers? + + Same calculus as any process pool: about core-count for + CPU-bound work (the default sizing in + :doc:`/guide/clustering`); more only if workers block on I/O — + though at that point you likely want plain ``trio`` tasks, not + processes. + +And because the pool is just SC code, every variation — bounded +submission, per-worker state, streaming partial results (see +:doc:`/guide/streaming`), nested pools — is a local edit to your +pool, not a feature request against an executor class B) + +An *async* pool, though? +************************ + +Yep: RPC targets must be async functions — the runtime rejects a +plain ``def`` with ``TypeError: ... must be an async function!``. +That's not zealotry, it's cancel-responsiveness: each worker is a +full ``trio`` runtime whose msg loop is what hears graceful cancel +requests, and a hot loop that never yields can't be (politely) +interrupted. + +Two practical consequences, + +- CPU-bound loops should checkpoint once in a while; note how + ``burn_cpu()`` in the next example sprinkles ``await + trio.sleep()`` calls so the worker stays responsive while still + pegging a core, +- if some sync call blocks a worker anyway you're still covered: + an unresponsive actor just rides the graceful-then-hard teardown + ladder from :doc:`/guide/cancellation` instead of acking its + cancel — slower, but never a zombie. + +Run a func in a process +----------------------- + +Even a pool can be overkill; "run this one async func in a +subprocess and give me the result" is a one-liner via +:meth:`tractor.ActorNursery.run_in_actor`, + +.. literalinclude:: ../../examples/parallelism/single_func.py + :caption: examples/parallelism/single_func.py + :language: python + +``run_in_actor()`` is a *convenience wrapper* — spawn an actor, run +exactly one task in it, reap on result — not the core spawning +model (that's :meth:`tractor.ActorNursery.start_actor` plus +:meth:`tractor.Portal.open_context`; see :doc:`/guide/context`). +But for this fire-and-collect shape it's exactly the right amount +of typing. + +As the module docstring suggests, run it under a process-tree +monitor to watch the child appear and get reaped, + +.. code:: bash + + $TERM -e watch -n 0.1 "pstree -a $$" \ + & python examples/parallelism/single_func.py \ + && kill $! + +You'll see a core get burned in both parent and child — real +parallelism, no GIL sharing, since these are processes (i.e. +*non-shared-memory threads*). + +When all you have is sync code +------------------------------ + +Honesty corner: if your workload is purely *synchronous* functions +and you've zero need for IPC dialogs, streaming, daemons or +supervision trees — i.e. you really do just want +"``ProcessPoolExecutor`` but ``trio``-native" — the smaller, +focused `trio-parallel`_ project may serve you better. ``tractor`` +happily covers the use case (as above) but brings a whole runtime +along for the ride. (And when blocking I/O — not the GIL — is the +actual problem, plain in-process :func:`trio.to_thread.run_sync` +may be all you ever needed.) + +And to *see* that runtime's process-management story — a per-core +fleet self-destructing with zero zombies left behind — go run +``examples/parallelism/we_are_processes.py``, walked through in +the :doc:`/start/quickstart`. + +.. seealso:: + + - :doc:`/guide/clustering` — the one-liner flat-cluster + convenience (``open_actor_cluster()``) for when even a + hand-rolled pool is too much typing, + - :doc:`/guide/cancellation` — why pool teardown is bulletproof + (graceful-then-hard escalation, no zombies), + - :doc:`/guide/context` — the core per-task API your pool + workers can graduate to. + +.. _trio-parallel: https://github.com/richardsheridan/trio-parallel diff --git a/docs/guide/rpc.rst b/docs/guide/rpc.rst new file mode 100644 index 000000000..6c5d8fa00 --- /dev/null +++ b/docs/guide/rpc.rst @@ -0,0 +1,177 @@ +RPC: calling into other actors +============================== +Every spawn call from :doc:`/guide/spawning` hands you back +a :class:`~tractor.Portal`: a live handle for calling into +another actor's **memory domain**. The name is borrowed from +``trio``'s portal concept — an object you use to submit work +*into* a separate concurrency domain — except here that domain +is a whole other process. + +.. d2:: diagrams/runtime_stack.d2 + :margin: + :caption: The layers a ``portal.run()`` request rides through. + :alt: app, tractor runtime, IPC channel and OS process layers + +There are **no proxy objects** and no special calling +conventions: you pass a plain function reference plus keyword +args, and Python's normal ``await``-able semantics apply. The +function just happens to *run somewhere else*; from the calling +task it looks as though it was called locally. And since this is +all structured concurrency (SC) under the hood, the remote task +runs inside the callee's supervised task tree while its result +— or its failure, as a boxed +:exc:`~tractor.RemoteActorError` — always comes back to *you*. + +``Portal.run()``: pass the function, not a string +------------------------------------------------- +:meth:`~tractor.Portal.run` schedules an async function as +a **new task** in the remote actor and waits on its result: + +.. code:: python + + async with tractor.open_nursery() as an: + portal = await an.start_actor( + 'service', + enable_modules=[__name__], + ) + answer = await portal.run(movie_theatre_question) + +The rules of engagement: + +- the target must be an **async function** and its defining + module must be in the callee's ``enable_modules`` allowlist, + else an :exc:`~tractor.ModuleNotExposed` error is relayed + back (see :doc:`/guide/spawning` for the capability-allowlist + story). +- arguments are passed **by keyword only**; they ride the IPC + layer as msgspec_-encoded msgs, so keep them serializable. +- every call schedules a *fresh* task remotely — call it twice + and the callee runs two tasks, each supervised in its own + right. +- remote exceptions re-raise locally as + :exc:`~tractor.RemoteActorError` with the original type + preserved via ``.boxed_type``. + +.. note:: + + Passing dotted-path *strings* to ``run()`` is an ancient, + deprecated form; always pass the function reference. If you + really need name-based addressing use ``run_from_ns()`` + below. + +Namespaced daemons: ``run_from_ns()`` +------------------------------------- +Sometimes the calling process can't (or shouldn't) import the +target function — think a long-running rpc-daemon serving +modules your client never loads. For that, +:meth:`~tractor.Portal.run_from_ns` takes the explicit +namespace path: + +.. code:: python + + await portal.run_from_ns('mypkg.service', 'ping') + +This is literally how ``.run()`` works underneath: the pair is +encoded as a ``'mod.path:func'`` style msg and resolved against +the callee's enabled modules. + +One special namespace exists: ``'self'`` resolves to the remote +:class:`~tractor.Actor` instance, i.e. the runtime itself. It's +how internal machinery (cancel requests, registry ops) travels; +don't build your app on it. + +One-shot results: ``wait_for_result()`` +--------------------------------------- +A portal returned from +:meth:`~tractor.ActorNursery.run_in_actor` has exactly one +"main" task running remotely; that task's ``return`` value is +delivered as the portal's *final result*: + +.. code:: python + + portal = await an.run_in_actor(fib, n=10) + final = await portal.wait_for_result() + +Semantics worth knowing: + +- it blocks until the remote task returns, re-raising any + remote error in the usual boxed form. +- once resolved it's idempotent: later calls return the same + cached value. +- a *daemon* portal (from ``start_actor()``) has no main task, + so there's no final result to wait for: you'll get a warning + plus a ``NoResult`` sentinel. Results of individual daemon + calls come straight back from each ``await portal.run()``. + +Pure RPC daemons: ``run_daemon()`` +---------------------------------- +When a process's *only* job is to sit at the root of its own +tree and serve RPC, skip the boilerplate with +:func:`tractor.run_daemon`: + +.. code:: python + + import tractor + + tractor.run_daemon( + ['mypkg.service'], + name='service', + ) + +It's a blocking convenience (it calls ``trio.run()`` for you): +boot a root actor with the given modules enabled for RPC, then +sleep until cancelled. Pair it with the discovery system — +:func:`tractor.find_actor` / :func:`tractor.wait_for_actor` +from a *separate* program — and you've got a tiny service +architecture with zero framework ceremony; see +``examples/service_daemon_discovery.py`` for the full pattern. + +Fan-out: RPC through nested trees +--------------------------------- +Portals compose. An RPC task is just a ``trio`` task, so it can +open its own :class:`~tractor.ActorNursery` and portal into +*its* children — one inbound call fanning out into a whole +sub-tree of work. The mid-tier function from the nested-tree +example: + +.. literalinclude:: ../../examples/nested_actor_tree.py + :caption: examples/nested_actor_tree.py (supervisor fan-out) + :language: python + :pyobject: fan_out_squares + +The root portals into the ``supervisor`` actor; the +supervisor's RPC task spawns the leaf workers, portals into +each, and returns the combined result back up. Failures at any +depth relay hop-by-hop as boxed errors, and cancelling the root +call tears down the entire sub-tree — SC, transitively. + +When to graduate to ``Context`` +------------------------------- +``portal.run()`` is great for one-shot, request-response calls. +Reach for :meth:`~tractor.Portal.open_context` with an +``@tractor.context`` endpoint as soon as you want: + +- a long-lived dialog with state held on both sides, +- bidirectional streaming via ``ctx.open_stream()``, +- typed payload contracts (``pld_spec``) enforced at the msg + layer, +- or *task-scoped* cancellation: ``Context.cancel()`` cancels + just the linked remote task, whereas + :meth:`~tractor.Portal.cancel_actor` nukes the **entire** + remote runtime and its process. + +In fact the source plans for ``Portal.run()`` itself to be +rebuilt on top of ``open_context()`` — contexts *are* the core +inter-actor protocol. Take the full tour in +:doc:`/guide/context`. + +.. seealso:: + + - :doc:`/guide/spawning` — where portals come from and how + their actors are supervised. + - :doc:`/guide/context` — the structured cross-actor task + API: handshake, streaming, typed payloads. + - :doc:`/guide/cancellation` — what happens to in-flight RPC + when trees get torn down. + +.. _msgspec: https://jcristharif.com/msgspec/ diff --git a/docs/guide/spawning.rst b/docs/guide/spawning.rst new file mode 100644 index 000000000..f1bbf26f9 --- /dev/null +++ b/docs/guide/spawning.rst @@ -0,0 +1,299 @@ +Spawning actors +=============== +If you know trio_ you know the drill: you don't get to launch +a task off into the void, you open a nursery_, the nursery owns +the task, and the block can't exit until every child is done. +That discipline is `structured concurrency`_ (SC) — see the +seminal `blog post`_ if you haven't yet — and it's the whole +religion around here. + +``tractor`` applies that exact discipline to **processes**: an +:class:`~tractor.ActorNursery` is a *process nursery*. Every +"task" it starts is a fresh Python process running its own +``trio.run()``-scheduled task tree; we call each one a +``trio``-"*actor*". Parents must wait on (and clean up after) +their children, transitively, all the way down the tree. + +.. d2:: diagrams/actor_tree.d2 + :caption: A process tree of ``trio``-task-trees. + :alt: a nested actor tree where every parent supervises its children + +Though a "process nursery" differs in complexity (and slightly +in semantics) from a single-threaded task nursery, most of the +interface is the same. The main difference is that each spawned +child contains a full, *parallel-executing* ``trio`` task tree. +The following super powers ensue: + +- tasks started in a child actor are completely independent of + tasks started in the current process; they execute in + **parallel** and are scheduled by their own actor's ``trio`` + run loop. +- tasks scheduled in a remote process still maintain an SC + protocol *across memory boundaries* using a so called + "SC dialogue protocol" which keeps task-hierarchy lifetimes + linked across the IPC layer. +- a remote task can fail and have that failure relayed back to + the caller task (living in some other actor) as a serialized + :exc:`~tractor.RemoteActorError`; no spawned process or RPC + task can ever just go off on its own. + +Opening a (process) nursery +--------------------------- +:func:`tractor.open_nursery` is the entrypoint: + +.. code:: python + + async def main(): + async with tractor.open_nursery() as an: + ... # spawn some actors B) + + trio.run(main) + +Notice there's no runtime-boot ceremony: if no actor runtime is +up yet (i.e. you're in a plain old Python process), +``open_nursery()`` *implicitly* enters +:func:`tractor.open_root_actor` for you, making this process the +**root actor** of a new tree. Any extra keyword args you pass +are proxied straight through to ``open_root_actor()``, so the +runtime config lives wherever you open your first nursery: + +.. code:: python + + async with tractor.open_nursery( + loglevel='info', + debug_mode=True, # crash-to-REPL for the whole tree + ) as an: + ... + +If you want the runtime up *without* spawning anything (or you +prefer the config to be loudly explicit) enter +``open_root_actor()`` yourself first; the nursery will detect +the running runtime and skip the implicit boot. Either way, +nesting a second root inside an existing tree is an error. + +Inside a *subactor* the same call just works: any actor may open +nurseries of its own, which is how you get arbitrarily deep +trees (more on that below). + +``start_actor()``: daemons that live until cancelled +---------------------------------------------------- +:meth:`~tractor.ActorNursery.start_actor` is **the** core +spawning primitive. It starts a *daemon* actor: a process with +no designated "main task" besides the runtime itself. It boots, +registers with its parent, and then sits there serving RPC +requests until somebody cancels it. You get back a +:class:`~tractor.Portal` for doing exactly that kind of +somebody-ing: + +.. literalinclude:: ../../examples/actor_spawning_and_causality_with_daemon.py + :caption: examples/actor_spawning_and_causality_with_daemon.py + :language: python + +What's going on here? + +- ``start_actor('frank', enable_modules=[__name__])`` forks off + a new process, boots a ``tractor`` runtime inside it, and + allows it to serve functions from the current module (see the + allowlist section below). +- each ``await portal.run(...)`` schedules a *new* task in + frank's task tree and waits on its result — the full RPC story + lives in :doc:`/guide/rpc`. +- frank has no main task to complete, so without the final + ``await portal.cancel_actor()`` the nursery block would wait + on him **forever**. Daemon lifetimes are *yours* to end; that + explicitness is the point. + +``run_in_actor()``: quick one-shot parallelism +---------------------------------------------- +:meth:`~tractor.ActorNursery.run_in_actor` is the convenience +wrapper: spawn an actor, run exactly one async function in it, +then reap the process as soon as the result arrives. + +.. code:: python + + async with tractor.open_nursery() as an: + portal = await an.run_in_actor(burn_cpu) + # burn rubber in the parent too... + await burn_cpu() + total = await portal.wait_for_result() + +A few details worth knowing: + +- the actor is named after the function unless you pass + ``name='something_cuter'``. +- the function's module is auto-added to the child's + ``enable_modules`` allowlist. +- extra ``**kwargs`` are forwarded to the function itself. +- the child is *auto-cancelled* once its "main" result lands; + at nursery exit these run-once children are always reaped + first (causality_ is paramount!). + +.. note:: + + ``run_in_actor()`` is a convenience, **not** the core model. + The source literally marks it for an eventual rebuild as + a thin "hilevel" wrapper on top of + :meth:`~tractor.Portal.open_context` (the modern inter-actor + task API). Teach your fingers to use it for quick + fire-and-collect parallelism — think a per-function + trio-parallel_ style one-shot — and reach for + ``start_actor()`` + ``open_context()`` for anything + long-lived, stateful or streaming + (:doc:`/guide/context`). + +Actor lifetimes and teardown order +---------------------------------- +So we have two lifetime flavors: + +- **run-once** (``run_in_actor()``): lives exactly as long as + its single task; reaped the moment its result (or error) + arrives. +- **daemon** (``start_actor()``): lives until *someone* cancels + it — an explicit ``await portal.cancel_actor()``, a bulk + ``await an.cancel()``, or the one-cancels-all strategy kicking + in on error. + +On a clean exit of the nursery block the teardown order is: + +1. the nursery waits on every run-once actor's final result; + any errors from these are raised immediately so your code + (acting as supervisor) gets first crack at handling them. +2. then it waits on daemon actors — **indefinitely**. If you + spawned a daemon, you own its lifetime. + +When a child *is* cancelled, teardown is graceful-first per SC +discipline: the runtime sends an IPC cancel request and gives +the child a bounded window to ack; only when a child is too +slow does the nursery escalate to an OS-level hard kill of the +process. There is no path where a child is silently left +running: + + ``tractor`` tries to protect you from zombies, no matter + what. If you can create zombie child processes (without + using a system signal) it **is a bug**. + +Per-process cleanup hooks +************************* +Need something torn down when an actor's runtime exits, no +matter how it exits? Every actor carries +a process-global :class:`contextlib.ExitStack` at +``Actor.lifetime_stack`` which is closed at the very end of +runtime teardown: + +.. code:: python + + db = await connect_db() + tractor.current_actor().lifetime_stack.callback(db.close) + +(A so-far under-advertised api — expect it to get more love.) + +When things blow up: one-cancels-all +------------------------------------ +The default (and currently only) supervision strategy is the +same one ``trio`` nurseries use: **one-cancels-all**. If your +nursery-block body errors, every child actor is cancelled. If +a child errors, the failure is relayed to the nursery as a +boxed :exc:`~tractor.RemoteActorError` (original type preserved +via ``.boxed_type``), all *other* children are cancelled, and +the error(s) re-raise locally — exactly like ``trio``, just +process-wide. Erlang-style alternative strategies are a long +standing roadmap item. + +The full story — how cancel requests relay across the tree, who +``.canceller`` was, debugging mid-teardown — lives in +:doc:`/guide/cancellation`. + +The module allowlist: ``enable_modules`` +---------------------------------------- +A subactor will only serve functions from modules its parent +*explicitly* enabled at spawn time: + +.. code:: python + + portal = await an.start_actor( + 'service', + enable_modules=['mypkg.service'], # or [__name__] + ) + +At child boot the runtime imports each listed module so inbound +RPC requests can resolve function references against it. Ask +a peer to run something from any *other* module and you get an +:exc:`~tractor.ModuleNotExposed` error relayed back — the child +never even looks the function up. + +Think of it as the first, deliberately coarse layer of +capability-style permissioning: if you don't hand an actor +a module, no peer can invoke anything inside it. (Finer-grained +capability-based messaging protocols are on the roadmap.) + +The ``enable_modules=[__name__]`` idiom — "let the child run +functions from the *current* module" — is what you'll use in +most scripts; bigger apps tend to pass dedicated service-module +paths instead. + +Per-child knobs +--------------- +Both spawn methods accept per-child config so one weird child +doesn't have to drag the whole tree along: + +- ``loglevel='cancel'`` — crank console logging for just this + subactor (the ``TRACTOR_LOGLEVEL`` env var overrides whatever + the *root* was passed, handy for test runs). +- ``debug_mode=True`` — arm the crash-handling REPL machinery + for just this child instead of tree-wide, i.e. the selective + flavor of ``open_nursery(debug_mode=True)``; see + :doc:`/guide/debugging` for the multi-process debugger tour. +- ``infect_asyncio=True`` — run the child with ``trio`` as an + ``asyncio`` guest, aka "infected asyncio" mode. +- ``enable_transports=['uds']`` — pick the IPC transport this + child should listen on (default ``'tcp'``). + +Trees all the way down +---------------------- +Since any actor can open an ``ActorNursery``, supervision trees +compose to arbitrary depth: a subactor can be a supervisor of +*its own* subactors, with every level holding the same SC +guarantees — error relay up, cancellation down, no orphans. + +.. literalinclude:: ../../examples/nested_actor_tree.py + :caption: examples/nested_actor_tree.py + :language: python + +Here the root spawns a ``supervisor`` actor whose RPC task opens +its *own* nursery and spawns the leaf workers; one call from the +root fans out through the middle layer and the aggregate comes +back up. Teardown ripples in reverse: the leaves are reaped when +the supervisor's nursery exits, the supervisor when the root +cancels it. + +Watching your tree grow +----------------------- +Actors are real processes, so your favorite system tools just +work. The house incantation runs any example beside a live +process-tree monitor:: + + $TERM -e watch -n 0.1 "pstree -a $$" \ + & python examples/nested_actor_tree.py \ + && kill $! + +Every subactor also sets its OS process title to a stable +``_subactor[@]`` marker, so ``htop``, +``ps`` and friends show *which actor is which* at a glance:: + + pgrep -af '_subactor\[' + +.. seealso:: + + - :doc:`/guide/rpc` — actually invoking functions through + all these portals you've been collecting. + - :doc:`/guide/context` — the structured, streaming-capable + inter-actor task API. + - :doc:`/guide/cancellation` — cross-actor cancellation and + error propagation semantics in depth. + +.. _trio: https://github.com/python-trio/trio +.. _nursery: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ +.. _causality: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#c-c-c-c-causality-breaker +.. _trio-parallel: https://github.com/richardsheridan/trio-parallel diff --git a/docs/guide/streaming.rst b/docs/guide/streaming.rst new file mode 100644 index 000000000..1c071268a --- /dev/null +++ b/docs/guide/streaming.rst @@ -0,0 +1,244 @@ +Cross-process streaming +======================= + +Spawning processes is the boring half of ``tractor``: the **real +cool stuff** is the native support for cross-process *streaming*. +Yes, you saw it here first — 2-way msg streams with reliable, +transitive setup/teardown semantics, wired straight into the +runtime's `structured concurrency`_ (SC) supervision machinery so +that *how* a stream ends is part of the protocol. + +No broker, no topic exchange, no IDL compiler. The IPC layer is a +deliberately "`cheap or nasty`_" `(un)protocol`_: a tiny set of +msgspec_-typed msgs over a transport (TCP or UDS today) with +payload typing opt-in per dialog — handshake msgs get the *nasty* +treatment (strict validation) while high-rate stream payloads +stay *cheap* (receiver-side checks only). See +:doc:`/guide/context` for the typed ``pld_spec`` contract bits. + +Two ways to stream +------------------ + +.. margin:: It's a ``trio.abc.Channel`` + + :class:`tractor.MsgStream` implements + :class:`trio.abc.Channel` — ``send()``, + ``receive()``, async-iteration, ``aclose()`` — + so trio-generic channel code drives an IPC + stream unchanged. + +- **Bidirectional, context-based**: open a + :class:`tractor.Context` to a peer task then enter + ``ctx.open_stream()`` for a full-duplex + :class:`tractor.MsgStream`. This is the modern core API, taught + end-to-end in :doc:`/guide/context`; we won't re-teach it here. + +- **One-way, portal-based**: point + :meth:`tractor.Portal.open_stream_from` at a plain async + generator fn in the peer actor. Legacy, but perfectly fine for + simple produce/consume pipelines — and it powers the classic + examples below. + +Rule of thumb: if the consumer ever needs to *talk back* — acks, +control msgs, a final result — use a context. If it's a pure +pipeline stage, either works and the one-way form is less typing. + +One-way streaming from an async generator +----------------------------------------- + +The OG api. Write an async generator in the target actor's +module; iterate its yields from the spawning side: + +.. literalinclude:: ../../examples/asynchronous_generators.py + :caption: examples/asynchronous_generators.py + :language: python + +Each ``yield`` crosses the process boundary as one msg and feeds +the parent's ``async for``. When the consumer ``break``\ s out +and exits the ``open_stream_from()`` block the far-end generator +task is cancelled for you: the producer's lifetime is *coupled to +the consumer's scope* so a one-way stream can never leak a remote +task. + +Any extra kwargs (``stream_data, seed=100`` style) are forwarded +to the remote generator's call, and a non-async-gen target is +rejected up front with a ``TypeError``. + +.. note:: + + No decorator required — any plain async-gen fn works. You may + still meet ``@tractor.stream`` in the wild; it's the legacy + marker for one-way endpoints and sticks around only for + compat (heads up: the param name ``ctx`` is reserved for + ``@context`` endpoints nowadays, so legacy fns should call + theirs ``stream``). New code wanting anything fancier than a + one-way pipe should use :func:`tractor.context` + + ``ctx.open_stream()``. + +.. warning:: + + One-way means one way: there's no sending *to* the generator + side and no graceful consumer-to-producer stop msg — the + teardown above is cancel-based. Needing upstream control flow + is the sign you've outgrown this API. + +A full-fledged streaming service +-------------------------------- + +Now let's get fancy: compose one-way streams through a nested +actor tree and you've got yourself a fan-in pipeline. + +.. d2:: diagrams/streaming_pipeline.d2 + :caption: Four actors, three streams, one deduped feed. + :alt: two streamer actors fan in to an aggregator then root + +.. literalinclude:: ../../examples/full_fledged_streaming_service.py + :caption: examples/full_fledged_streaming_service.py + :language: python + +What's going on? + +- the root actor spawns ``'aggregator'`` which opens its *own* + actor nursery and spawns ``'streamer_1'`` + ``'streamer_2'``: 4 + processes total, supervision nested two levels deep with zero + special casing. + +- ``aggregate()`` opens a one-way stream from each streamer and + fans both into a single :func:`trio.open_memory_channel` via + one local trio task per portal — in-actor fan-in riding trio's + built-in backpressure end-to-end. + +- duplicates get dropped via a ``set`` and the deduped sequence + is *re-yielded* upward: ``aggregate()`` is itself an async gen + being consumed over IPC by the root. Streams compose. + +- when the seed runs out the streamer gens finish, the memory + channel drains closed, the aggregator's gen returns and the + root's ``async for`` ends; ``await an.cancel()`` then reaps the + subtree. Every exit is awaited — if you can produce a zombie + process from this, it **is a bug**. + +Watch the tree breathe while it runs, using the README's +signature process-monitor incantation:: + + $TERM -e watch -n 0.1 "pstree -a $$" \ + & python examples/full_fledged_streaming_service.py \ + && kill $! + +No extra threads, no fancy semaphores, no futures; all we need is +``tractor``'s IPC. + +Two streams, one portal +----------------------- + +Every ``open_stream_from()`` call starts its *own* remote task — +even through the same portal — so two local consumer tasks can +independently stream the same generator fn concurrently, both +dialogs multiplexed over the single underlying IPC channel: + +.. literalinclude:: ../../examples/multiple_streams_one_portal.py + :caption: examples/multiple_streams_one_portal.py + :language: python + +The add-else-remove trick on the shared ``consumed`` list is the +proof: each value arrives in *both* streams, getting appended by +whichever task sees it first and removed by the other, so the +list always ends up empty. Two streams, same data, zero +interference. + +This works because every dialog is keyed by its own context id +(``Context.cid``) — any number of concurrent streams, contexts +and one-shot RPCs share a single underlying +:class:`tractor.Channel` per peer pair. + +Fan-out inside an actor: ``MsgStream.subscribe()`` +-------------------------------------------------- + +The inverse pattern: *one* IPC stream feeding *many* local tasks. +Instead of paying for N redundant cross-process streams, call +:meth:`tractor.MsgStream.subscribe` to get a +``BroadcastReceiver`` — a tokio-style broadcast channel from +``tractor.trionics`` — which copies every received value to each +subscribed task: + +.. literalinclude:: ../../examples/streaming_broadcast_fanout.py + :caption: examples/streaming_broadcast_fanout.py + :language: python + +Each task entering ``stream.subscribe()`` receives its own copy +of everything sent from that point on. The underlying stream +keeps pace with the *fastest* subscriber; a task falling more +than the buffered window behind has its next receive raise +``tractor.trionics.Lagged`` to say it lost data. + +The broadcast handle stays duplex btw: it proxies ``send()`` +through to the underlying stream, so each subscriber task can +keep talking upstream while consuming its fan-out copy. + +.. warning:: + + ``.subscribe()`` is **idempotent and non-reversible**: the + first call permanently swaps the stream's receive machinery + over to the internally allocated broadcaster. There's no + un-subscribing back to the raw stream, so make sure you're ok + with the (theoretical) overhead before opting in. + +Consuming: ``async for`` and friends +------------------------------------ + +``async for msg in stream:`` is just sugar over repeated +``await stream.receive()``. The receive-side surface: + +- ``receive()`` — next msg, or raises :exc:`trio.EndOfChannel` + on a graceful far-end close (``async for`` translates that + into a clean loop exit for you). + +- ``receive_nowait()`` — opportunistic, non-blocking drain. + +- ``closed`` — property flagging an already-ended stream. + +Send-side it's just ``await stream.send(data)`` — one ``Yield`` +msg per call carrying any msgspec_-encodable payload (or +whatever your ``pld_spec`` permits, see :doc:`/guide/context`). + +End-of-stream: close vs. cancel +------------------------------- + +How a stream ends is part of the protocol; the runtime keeps the +polite case and the violent case distinct: + +- **graceful close**: the far side exits its stream block, its + async gen returns, or it calls ``await stream.aclose()``. A + ``Stop`` msg is sent so your ``async for`` simply ends + (``StopAsyncIteration``, via :exc:`trio.EndOfChannel` under the + hood). A normal, non-error ending — the dialog's result phase + proceeds as usual. + +- **cancel or error**: no ``Stop`` is sent. Instead the + cancel/error itself is relayed so the far end *knows* the + dialog did not end on purpose and raises accordingly — a + :exc:`tractor.ContextCancelled`, a boxed + :exc:`tractor.RemoteActorError`, etc. See the cancellation + section of :doc:`/guide/context` for exactly who raises what. + +Tying it together: every ``MsgStream`` is **one-shot use**. Both +endings are final — once closed a stream can't be re-opened and +the supported "retry" is opening a fresh :class:`tractor.Context` +(they're cheap). + +.. seealso:: + + - :doc:`/guide/context` — the full ``Context`` lifecycle: the + handshake, results, cancellation semantics and the + overrun/backpressure knobs. + + - :class:`tractor.MsgStream` and + :meth:`tractor.Portal.open_stream_from` API docs. + + - The zguide chapters our wire philosophy is named after: + "`cheap or nasty`_" and `(un)protocol`_\ s. + +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _cheap or nasty: https://zguide.zeromq.org/docs/chapter7/#The-Cheap-or-Nasty-Pattern +.. _(un)protocol: https://zguide.zeromq.org/docs/chapter7/#Unprotocols +.. _msgspec: https://jcristharif.com/msgspec/ diff --git a/docs/guide/testing.rst b/docs/guide/testing.rst new file mode 100644 index 000000000..d07415b8b --- /dev/null +++ b/docs/guide/testing.rst @@ -0,0 +1,256 @@ +Testing tips +============ + +``tractor``'s test suite is a different kind of beast than your +average single-proc pytest run: nearly every test spawns a real +**process tree**, hammers on cancellation under structured +concurrency (SC), and tears the whole thing down again — hundreds +of times per session. This page collects the tips, knobs and +one-liners that make hacking on (and with) the suite pleasant. + +Running the suite +----------------- + +This is a uv_-managed project, so after cloning it's just:: + + uv sync --dev + uv run pytest tests/ + +Expect a *lot* of process churn; the suite is effectively a +rolling chaos exercise for the runtime. + +The classic fix-iterate loop when something breaks:: + + # stop at the first failure + uv run pytest tests/ -x + + # then iterate on just the failures til green + uv run pytest --lf -x + +``--lf`` (last-failed) re-runs only what failed previously, so +combined with ``-x`` you get a tight one-test-at-a-time repair +loop. + +Suite-specific flags +******************** + +The repo auto-loads the bundled ``tractor._testing.pytest`` plugin +(via ``addopts`` in ``pyproject.toml``) which adds a few extra +flags: + +- ``--spawn-backend ``: pick the process spawn backend for + the session (default ``'trio'``); same keys as the + ``start_method`` runtime argument, + +- ``--tpt-proto [...]``: which IPC transport(s) opting-in + suites should run against, eg. ``--tpt-proto uds``, + +- ``--tpdb`` / ``--debug-mode``: flip on the ``debug_mode`` + fixture so debugger-aware tests boot their trees with the + crash-REPL enabled, + +- ``--enable-stackscope``: install the ``SIGUSR1`` task-tree dump + handler in pytest *and* every spawned subactor — much lighter + than a full debug-mode run when you only need stack visibility + during a hang hunt, + +- ``--ll `` / ``--tl ``: console loglevels; + ``--tl`` targets the ``tractor``-as-runtime logger and accepts + a per-subsystem spec like ``'devx:runtime,trionics:cancel'``. + +Watch the tree grow +------------------- + +The single most useful trick while the suite (or any ``tractor`` +app) runs: keep a live ``pstree`` view going in a side terminal:: + + watch -n 0.1 "pstree -a $(pgrep -f pytest)" + +You'll see actor processes pop in and out of existence as each +test builds and reaps its tree. Launch it *after* pytest is up +(the pid is substituted once, at ``watch`` startup). + +Every subactor also sets its OS process title (via +``setproctitle``) to ``_subactor[@]`` so the +tree view shows *which actor is which* at a glance — and targeted +greps stay easy:: + + pgrep -af '_subactor\[' + +For a single example script, the repo's signature incantation +spawns the watcher alongside your program and cleans it up after:: + + $TERM -e watch -n 0.1 "pstree -a $$" \ + & python examples/parallelism/single_func.py \ + && kill $! + +Env-var knobs +------------- + +Two env-vars override their corresponding runtime arguments +*globally* — no application (or test) code changes required: + +``TRACTOR_SPAWN_METHOD`` + Wins over any caller-passed ``start_method`` so you can drive + the whole suite (or any app) under a different spawn backend:: + + TRACTOR_SPAWN_METHOD=mp_spawn uv run pytest tests/ -x + +``TRACTOR_LOGLEVEL`` + Wins over any caller-passed ``loglevel``; crank (or silence) + runtime console verbosity wholesale:: + + TRACTOR_LOGLEVEL=cancel uv run pytest tests/ -x -s + +``TRACTOR_ENABLE_STACKSCOPE`` + Force-install the ``SIGUSR1`` task-tree dump handler in every + actor, debug-mode or not; then + ``pkill --signal SIGUSR1 -f `` dumps every + actor's live ``trio`` task tree. + +Debug mode vs. pytest capture +----------------------------- + +The tree-wide crash-to-REPL experience (``debug_mode=True`` plus +``await tractor.pause()``) requires a **real tty**, and pytest's +default output capturing swallows exactly that. When you want to +interact with the REPL from inside a test run, disable capture:: + + uv run pytest tests/test_foo.py -x -s + +(``-s`` is shorthand for ``--capture=no``.) + +Tests should request the ``debug_mode`` fixture (driven by the +``--tpdb`` flag) rather than hard-coding it, so that normal CI +runs stay non-interactive. + +For *automated* REPL interaction — asserting on prompt output, +sending debugger commands — you can't just turn capture off; +instead do what ``tests/devx/`` does: drive a child Python program +through pexpect_ on a real pseudo-tty and pattern-match the +``(Pdb+)`` prompts. See ``tests/devx/test_debugger.py`` for many +worked patterns. + +Examples *are* tests +-------------------- + +Every script under ``examples/`` is run as a subprocess by +``tests/test_docs_examples.py``; since these docs +``literalinclude`` those same scripts, the code you read here is +CI-verified on every push and can never silently rot B) + +Conventions when adding a new example: + +- make it a standalone runnable script with the usual guard:: + + if __name__ == '__main__': + trio.run(main) + +- it must exit cleanly (returncode ``0``) within the per-example + timeout (~16s locally, with headroom auto-added in CI and under + cpu-freq scaling) — keep sleeps short, + +- any stderr line containing ``Error`` fails the test, so silence + or assert-around expected error output, + +- don't crank ``tractor`` logging inside an example: subprocess + pipe **backpressure can deadlock** the run (ask us how we + know..), + +- filenames starting with ``_`` are skipped (the WIP convention), + as are the special subdirs (``debugging/``, ``integration/``, + ``advanced_faults/``, ``trio/``) which are driven by their own + dedicated suites instead. + +Drop your script in, run the example suite, profit:: + + uv run pytest tests/test_docs_examples.py -x + +Zombie cleanup +-------------- + +First, the contract: ``tractor`` **always** reaps its children — +if you can create a zombie process (without resorting to +untrappable signals) it **is a bug**, please report it! + +That said, while hacking on the *runtime itself* you can +definitely wedge things — a ``SIGKILL``-ed pytest, a half-broken +spawn backend — and strand subactor procs plus their shm segments +and UDS socket files. The repo ships a dedicated cleanup tool:: + + uv run scripts/tractor-reap --shm --uds + +It's SC-polite even as a reaper: matched processes get ``SIGINT`` +first with a bounded grace window — so actor runtimes can run +their ``trio`` teardown paths — escalating to ``SIGKILL`` only as +a last resort. The ``--shm`` sweep unlinks ``/dev/shm/`` segments +that no live process has open (it leans on psutil_, already in +your dev venv, to check live mappings and fds) and ``--uds`` +clears socket files whose binder pid is dead. + +Testing your own ``tractor`` app +-------------------------------- + +The same plugin the suite uses ships in the package, so your +project can load it too:: + + [tool.pytest.ini_options] + addopts = ['-p tractor._testing.pytest'] + +That buys you the CLI flags above plus a set of fixtures — +``loglevel``, ``debug_mode``, ``reg_addr`` (a session-unique +registrar address so concurrent runs and other live ``tractor`` +apps on the host can't cross-talk) — and the ``@tractor_test`` +decorator: + +.. code:: python + + import tractor + from tractor._testing import tractor_test + + @tractor_test + async def test_my_service( + reg_addr: tuple, + loglevel: str, + ): + # already inside a root actor's trio task! + async with tractor.open_nursery() as an: + ... + +The decorator boots a root actor around your (async) test fn, +wires any of the special fixtures you declare (``reg_addr``, +``loglevel``, ``start_method``, ``debug_mode``) into +``open_root_actor()``, and runs the body as the root-most task +under a wall-clock ``trio.fail_after()`` guard. + +General advice that has served this suite well: + +- bound waits with ``trio.fail_after()`` *inside* tests; global + pytest timeout plugins interact badly with multi-process + ``trio`` teardown, + +- use the ``reg_addr`` fixture (or otherwise randomize your + registry addrs) so leftover registrars from prior runs can't + contaminate lookups, + +- assert on **structured outcomes** — eg. + ``RemoteActorError.boxed_type`` or + ``ContextCancelled.canceller`` — not on log text. + +.. note:: + ``tractor._testing`` is still an underscore-internal namespace: + shipped and handy, but its API may shift between alpha + releases. + +(This page exists thanks to the ask in `#126`_.) + +.. seealso:: + - :doc:`/guide/discovery` — how registrar wiring (the thing + ``reg_addr`` randomizes) works in the runtime proper. + - :doc:`/project/dev-tips` — contributor-oriented extras: + releases, log-system tracing, tree-monitoring recipes. + +.. _uv: https://docs.astral.sh/uv/ +.. _pexpect: https://pexpect.readthedocs.io/en/stable/ +.. _psutil: https://psutil.readthedocs.io/en/latest/ +.. _#126: https://github.com/goodboy/tractor/issues/126 diff --git a/docs/index.rst b/docs/index.rst index 4ff198f57..98759aeae 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,612 +1,154 @@ -.. tractor documentation master file, created by - sphinx-quickstart on Sun Feb 9 22:26:51 2020. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -``tractor`` -=========== - -A `structured concurrent`_, async-native "`actor model`_" built on trio_ and multiprocessing_. - -.. toctree:: - :maxdepth: 1 - :caption: Contents: - -.. _actor model: https://en.wikipedia.org/wiki/Actor_model -.. _trio: https://github.com/python-trio/trio -.. _multiprocessing: https://en.wikipedia.org/wiki/Multiprocessing -.. _trionic: https://trio.readthedocs.io/en/latest/design.html#high-level-design-principles -.. _async sandwich: https://trio.readthedocs.io/en/latest/tutorial.html#async-sandwich -.. _structured concurrent: https://trio.discourse.group/t/concise-definition-of-structured-concurrency/228 - - -``tractor`` is an attempt to bring trionic_ `structured concurrency`_ to -distributed multi-core Python; it aims to be the Python multi-processing -framework *you always wanted*. - -``tractor`` lets you spawn ``trio`` *"actors"*: processes which each run -a ``trio`` scheduled task tree (also known as an `async sandwich`_). -*Actors* communicate by exchanging asynchronous messages_ and avoid -sharing any state. This model allows for highly distributed software -architecture which works just as well on multiple cores as it does over -many hosts. - -The first step to grok ``tractor`` is to get the basics of ``trio`` down. -A great place to start is the `trio docs`_ and this `blog post`_. - -.. _messages: https://en.wikipedia.org/wiki/Message_passing -.. _trio docs: https://trio.readthedocs.io/en/latest/ -.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ -.. _structured concurrency: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ - - -Install -------- -No PyPi release yet! - -:: - - pip install git+git://github.com/goodboy/tractor.git - - -Feel like saying hi? +.. image:: _static/tractor_logo_side.svg + :class: hero-logo + :align: center + :alt: tractor + +tractor +======= +**distributed structured concurrency**: a +multi-processing runtime built on (and shaped +entirely like) trio_. + +``tractor`` provides parallelism via ``trio`` +*"actors"*: independent Python **processes** (ie. +*non-shared-memory threads*) each running a ``trio`` +task tree, all composed into a *distributed +supervision tree* with end-to-end `structured +concurrency`_ (SC) — spawning, cancellation, error +propagation and teardown that work **across +processes** (and hosts) exactly the way they work +across tasks. + +.. margin:: tl;dr + + It's **just** ``trio``, but with nurseries that + spawn *processes* and streams that cross them. If + you can read a ``trio`` program you can read a + ``tractor`` one — that's the whole pitch. + +Sixty seconds of why -------------------- -This project is very much coupled to the ongoing development of -``trio`` (i.e. ``tractor`` gets all its ideas from that brilliant -community). If you want to help, have suggestions or just want to -say hi, please feel free to ping me on the `trio gitter channel`_! - -.. _trio gitter channel: https://gitter.im/python-trio/general - - - -Philosophy ----------- -Our tenets non-comprehensively include: - -- strict adherence to the `concept-in-progress`_ of *structured concurrency* -- no spawning of processes *willy-nilly*; causality_ is paramount! -- (remote) errors `always propagate`_ back to the parent supervisor -- verbatim support for ``trio``'s cancellation_ system -- `shared nothing architecture`_ -- no use of *proxy* objects or shared references between processes -- an immersive debugging experience -- anti-fragility through `chaos engineering`_ - -``tractor`` is an actor-model-*like* system in the sense that it adheres -to the `3 axioms`_ but does not (yet) fulfil all "unrequirements_" in -practise. It is an experiment in applying `structured concurrency`_ -constraints on a parallel processing system where multiple Python -processes exist over many hosts but no process can outlive its parent. -In `erlang` parlance, it is an architecture where every process has -a mandatory supervisor enforced by the type system. The API design is -almost exclusively inspired by trio_'s concepts and primitives (though -we often lag a little). As a distributed computing system `tractor` -attempts to place sophistication at the correct layer such that -concurrency primitives are powerful yet simple, making it easy to build -complex systems (you can build a "worker pool" architecture but it's -definitely not required). There is first class support for inter-actor -streaming using `async generators`_ and ongoing work toward a functional -reactive style for IPC. - -.. warning:: ``tractor`` is in alpha-alpha and is expected to change rapidly! - Expect nothing to be set in stone. Your ideas about where it should go - are greatly appreciated! - -.. _concept-in-progress: https://trio.discourse.group/t/structured-concurrency-kickoff/55 -.. _3 axioms: https://en.wikipedia.org/wiki/Actor_model#Fundamental_concepts -.. _unrequirements: https://en.wikipedia.org/wiki/Actor_model#Direct_communication_and_asynchrony -.. _async generators: https://www.python.org/dev/peps/pep-0525/ -.. _always propagate: https://trio.readthedocs.io/en/latest/design.html#exceptions-always-propagate -.. _causality: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#c-c-c-c-causality-breaker -.. _shared nothing architecture: https://en.wikipedia.org/wiki/Shared-nothing_architecture -.. _cancellation: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-and-timeouts -.. _channels: https://en.wikipedia.org/wiki/Channel_(programming) -.. _chaos engineering: http://principlesofchaos.org/ - - -Examples --------- -Note, if you are on Windows please be sure to see the :ref:`gotchas -` section before trying these. - - -A trynamic first scene -********************** -Let's direct a couple *actors* and have them run their lines for -the hip new film we're shooting: - -.. literalinclude:: ../examples/a_trynamic_first_scene.py - -We spawn two *actors*, *donny* and *gretchen*. -Each actor starts up and executes their *main task* defined by an -async function, ``say_hello()``. The function instructs each actor -to find their partner and say hello by calling their partner's -``hi()`` function using something called a *portal*. Each actor -receives a response and relays that back to the parent actor (in -this case our "director" executing ``main()``). - - -Actor spawning and causality -**************************** -``tractor`` tries to take ``trio``'s concept of causal task lifetimes -to multi-process land. Accordingly, ``tractor``'s *actor nursery* behaves -similar to ``trio``'s nursery_. That is, ``tractor.open_nursery()`` -opens an ``ActorNursery`` which **must** wait on spawned *actors* to complete -(or error) in the same causal_ way ``trio`` waits on spawned subtasks. -This includes errors from any one actor causing all other actors -spawned by the same nursery to be cancelled_. - -To spawn an actor and run a function in it, open a *nursery block* -and use the ``run_in_actor()`` method: - -.. literalinclude:: ../examples/actor_spawning_and_causality.py - -What's going on? - -- an initial *actor* is started with ``trio.run()`` and told to execute - its main task_: ``main()`` - -- inside ``main()`` an actor is *spawned* using an ``ActorNusery`` and is told - to run a single function: ``cellar_door()`` - -- a ``portal`` instance (we'll get to what it is shortly) - returned from ``nursery.run_in_actor()`` is used to communicate with - the newly spawned *sub-actor* - -- the second actor, *some_linguist*, in a new *process* running a new ``trio`` task_ - then executes ``cellar_door()`` and returns its result over a *channel* back - to the parent actor - -- the parent actor retrieves the subactor's *final result* using ``portal.result()`` - much like you'd expect from a future_. - -This ``run_in_actor()`` API should look very familiar to users of -``asyncio``'s `run_in_executor()`_ which uses a ``concurrent.futures`` Executor_. - -Since you might also want to spawn long running *worker* or *daemon* -actors, each actor's *lifetime* can be determined based on the spawn -method: - -- if the actor is spawned using ``run_in_actor()`` it terminates when - its *main* task completes (i.e. when the (async) function submitted - to it *returns*). The ``with tractor.open_nursery()`` exits only once - all actors' main function/task complete (just like the nursery_ in ``trio``) - -- actors can be spawned to *live forever* using the ``start_actor()`` - method and act like an RPC daemon that runs indefinitely (the - ``with tractor.open_nursery()`` won't exit) until cancelled_ - -Here is a similar example using the latter method: - -.. literalinclude:: ../examples/actor_spawning_and_causality_with_daemon.py - -The ``enable_modules`` `kwarg` above is a list of module path -strings that will be loaded and made accessible for execution in the -remote actor through a call to ``Portal.run()``. For now this is -a simple mechanism to restrict the functionality of the remote -(and possibly daemonized) actor and uses Python's module system to -limit the allowed remote function namespace(s). - -``tractor`` is opinionated about the underlying threading model used for -each *actor*. Since Python has a GIL and an actor model by definition -shares no state between actors, it fits naturally to use a multiprocessing_ -``Process``. This allows ``tractor`` programs to leverage not only multi-core -hardware but also distribute over many hardware hosts (each *actor* can talk -to all others with ease over standard network protocols). - -.. _task: https://trio.readthedocs.io/en/latest/reference-core.html#tasks-let-you-do-multiple-things-at-once -.. _nursery: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning -.. _causal: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#causality -.. _cancelled: https://trio.readthedocs.io/en/latest/reference-core.html#child-tasks-and-cancellation -.. _run_in_executor(): https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.loop.run_in_executor -.. _Executor: https://docs.python.org/3/library/concurrent.futures.html#concurrent.futures.Executor - - -Cancellation -************ -``tractor`` supports ``trio``'s cancellation_ system verbatim. -Cancelling a nursery block cancels all actors spawned by it. -Eventually ``tractor`` plans to support different `supervision strategies`_ like ``erlang``. - -.. _supervision strategies: http://erlang.org/doc/man/supervisor.html#sup_flags - - -Remote error propagation -************************ -Any task invoked in a remote actor should ship any error(s) back to the calling -actor where it is raised and expected to be dealt with. This way remote actors -are never cancelled unless explicitly asked or there's a bug in ``tractor`` itself. - -.. literalinclude:: ../examples/remote_error_propagation.py - - -You'll notice the nursery cancellation conducts a *one-cancels-all* -supervisory strategy `exactly like trio`_. The plan is to add more -`erlang strategies`_ in the near future by allowing nurseries to accept -a ``Supervisor`` type. - -.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics -.. _erlang strategies: http://learnyousomeerlang.com/supervisors - - -IPC using *portals* -******************* -``tractor`` introduces the concept of a *portal* which is an API -borrowed_ from ``trio``. A portal may seem similar to the idea of -a RPC future_ except a *portal* allows invoking remote *async* functions and -generators and intermittently blocking to receive responses. This allows -for fully async-native IPC between actors. - -When you invoke another actor's routines using a *portal* it looks as though -it was called locally in the current actor. So when you see a call to -``await portal.run()`` what you get back is what you'd expect -to if you'd called the function directly in-process. This approach avoids -the need to add any special RPC *proxy* objects to the library by instead just -relying on the built-in (async) function calling semantics and protocols of Python. - -Depending on the function type ``Portal.run()`` tries to -correctly interface exactly like a local version of the remote -built-in Python *function type*. Currently async functions, generators, -and regular functions are supported. Inspiration for this API comes -`remote function execution`_ but without the client code being -concerned about the underlying channels_ system or shipping code -over the network. - -This *portal* approach turns out to be paricularly exciting with the -introduction of `asynchronous generators`_ in Python 3.6! It means that -actors can compose nicely in a data streaming pipeline. - -.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics - -Streaming -********* -By now you've figured out that ``tractor`` lets you spawn process based -*actors* that can invoke cross-process (async) functions and all with -structured concurrency built in. But the **real cool stuff** is the -native support for cross-process *streaming*. - - -Asynchronous generators -+++++++++++++++++++++++ -The default streaming function is simply an async generator definition. -Every value *yielded* from the generator is delivered to the calling -portal exactly like if you had invoked the function in-process meaning -you can ``async for`` to receive each value on the calling side. - -As an example here's a parent actor that streams for 1 second from a -spawned subactor: - -.. literalinclude:: ../examples/asynchronous_generators.py - -By default async generator functions are treated as inter-actor -*streams* when invoked via a portal (how else could you really interface -with them anyway) so no special syntax to denote the streaming *service* -is necessary. - - -Channels and Contexts -+++++++++++++++++++++ -If you aren't fond of having to write an async generator to stream data -between actors (or need something more flexible) you can instead use -a ``Context``. A context wraps an actor-local spawned task and -a ``Channel`` so that tasks executing across multiple processes can -stream data to one another using a low level, request oriented API. - -A ``Channel`` wraps an underlying *transport* and *interchange* format -to enable *inter-actor-communication*. In its present state ``tractor`` -uses TCP and msgpack_. - -As an example if you wanted to create a streaming server without writing -an async generator that *yields* values you instead define a decorated -async function: - -.. code:: python +Spawn one actor per core, crash the root on purpose, +and watch the runtime contain the blast: errors +propagate, *every* child is reaped, zero zombies — +guaranteed (it's a bug otherwise). - @tractor.stream - async def streamer(ctx: tractor.Context, rate: int = 2) -> None: - """A simple web response streaming server. - """ - while True: - val = await web_request('http://data.feed.com') +.. literalinclude:: ../examples/parallelism/we_are_processes.py + :caption: examples/parallelism/we_are_processes.py + :language: python - # this is the same as ``yield`` in the async gen case - await ctx.send_yield(val) +Like every snippet in these docs this file lives in +the repo's ``examples/`` dir and runs under CI — docs +code that can't rot. - await trio.sleep(1 / rate) +Dig in +------ +.. grid:: 1 2 2 3 + :gutter: 3 + .. grid-item-card:: Get started + :link: start/index + :link-type: doc -You must decorate the function with ``@tractor.stream`` and declare -a ``ctx`` argument as the first in your function signature and then -``tractor`` will treat the async function like an async generator - as -a stream from the calling/client side. + Install + your first actor tree in ~20 lines; + causality, daemons and the trynamic scene. -This turns out to be handy particularly if you have multiple tasks -pushing responses concurrently: + .. grid-item-card:: The big ideas + :link: explain/sc-distributed + :link-type: doc -.. code:: python + SC across processes, distilled — then the + runtime architecture under it. - async def streamer( - ctx: tractor.Context, - rate: int = 2 - ) -> None: - """A simple web response streaming server. - """ - while True: - val = await web_request(url) + .. grid-item-card:: Debug like a local + :link: guide/debugging + :link-type: doc - # this is the same as ``yield`` in the async gen case - await ctx.send_yield(val) + ``await tractor.pause()`` anywhere in the + tree: one terminal, every process, zero + socket-juggling. - await trio.sleep(1 / rate) + .. grid-item-card:: Streaming + contexts + :link: guide/context + :link-type: doc + Bidirectional, cancellation-safe msg streams + between any two actors. - @tractor.stream - async def stream_multiple_sources( - ctx: tractor.Context, - sources: List[str] - ) -> None: - async with trio.open_nursery() as n: - for url in sources: - n.start_soon(streamer, ctx, url) + .. grid-item-card:: Guides + :link: guide/index + :link-type: doc + RPC, supervision, clustering, "infected + asyncio", typed msging + more. -The context notion comes from the context_ in nanomsg_. + .. grid-item-card:: API reference + :link: api/index + :link-type: doc -.. _context: https://nanomsg.github.io/nng/man/tip/nng_ctx.5 -.. _msgpack: https://en.wikipedia.org/wiki/MessagePack - - - -A full fledged streaming service -++++++++++++++++++++++++++++++++ -Alright, let's get fancy. - -Say you wanted to spawn two actors which each pull data feeds from -two different sources (and wanted this work spread across 2 cpus). -You also want to aggregate these feeds, do some processing on them and then -deliver the final result stream to a client (or in this case parent) actor -and print the results to your screen: - -.. literalinclude:: ../examples/full_fledged_streaming_service.py - -Here there's four actors running in separate processes (using all the -cores on you machine). Two are streaming by *yielding* values from the -``stream_data()`` async generator, one is aggregating values from -those two in ``aggregate()`` (also an async generator) and shipping the -single stream of unique values up the parent actor (the ``'MainProcess'`` -as ``multiprocessing`` calls it) which is running ``main()``. - -.. _future: https://en.wikipedia.org/wiki/Futures_and_promises -.. _borrowed: - https://trio.readthedocs.io/en/latest/reference-core.html#getting-back-into-the-trio-thread-from-another-thread -.. _asynchronous generators: https://www.python.org/dev/peps/pep-0525/ -.. _remote function execution: https://codespeak.net/execnet/example/test_info.html#remote-exec-a-function-avoiding-inlined-source-part-i - - -Actor local (aka *process global*) variables -******************************************** -Although ``tractor`` uses a *shared-nothing* architecture between -processes you can of course share state between tasks running *within* -an actor (since a `trio.run()` runtime is single threaded). ``trio`` -tasks spawned via multiple RPC calls to an actor can modify -*process-global-state* defined using Python module attributes: - -.. code:: python - - - # a per process cache - _actor_cache: dict[str, bool] = {} - - - def ping_endpoints(endpoints: List[str]): - """Start a polling process which runs completely separate - from our root actor/process. - - """ - - # This runs in a new process so no changes # will propagate - # back to the parent actor - while True: - - for ep in endpoints: - status = await check_endpoint_is_up(ep) - _actor_cache[ep] = status - - await trio.sleep(0.5) - - - async def get_alive_endpoints(): - - nonlocal _actor_cache - - return {key for key, value in _actor_cache.items() if value} - - - async def main(): - - async with tractor.open_nursery() as n: - - portal = await n.run_in_actor(ping_endpoints) - - # print the alive endpoints after 3 seconds - await trio.sleep(3) - - # this is submitted to be run in our "ping_endpoints" actor - print(await portal.run(get_alive_endpoints)) - - -You can pass any kind of (`msgpack`) serializable data between actors using -function call semantics but building out a state sharing system per-actor -is totally up to you. - - -Service Discovery -***************** -Though it will be built out much more in the near future, ``tractor`` -currently keeps track of actors by ``(name: str, id: str)`` using a -special actor called the *arbiter*. Currently the *arbiter* must exist -on a host (or it will be created if one can't be found) and keeps a -simple ``dict`` of actor names to sockets for discovery by other actors. -Obviously this can be made more sophisticated (help me with it!) but for -now it does the trick. - -To find the arbiter from the current actor use the ``get_arbiter()`` function and to -find an actor's socket address by name use the ``find_actor()`` function: - -.. literalinclude:: ../examples/service_discovery.py - -The ``name`` value you should pass to ``find_actor()`` is the one you passed as the -*first* argument to either ``trio.run()`` or ``ActorNursery.start_actor()``. - - -Running actors standalone -************************* -You don't have to spawn any actors using ``open_nursery()`` if you just -want to run a single actor that connects to an existing cluster. -All the comms and arbiter registration stuff still works. This can -somtimes turn out being handy when debugging mult-process apps when you -need to hop into a debugger. You just need to pass the existing -*arbiter*'s socket address you'd like to connect to: - -.. code:: python - - import trio - import tractor - - async def main(): - - async with tractor.open_root_actor( - arbiter_addr=('192.168.0.10', 1616) - ): - await trio.sleep_forever() - - trio.run(main) - - -Choosing a process spawning backend -*********************************** -``tractor`` is architected to support multiple actor (sub-process) -spawning backends. Specific defaults are chosen based on your system -but you can also explicitly select a backend of choice at startup -via a ``start_method`` kwarg to ``tractor.open_nursery()``. - -Currently the options available are: - -- ``trio``: a ``trio``-native spawner which is an async wrapper around ``subprocess`` -- ``spawn``: one of the stdlib's ``multiprocessing`` `start methods`_ -- ``forkserver``: a faster ``multiprocessing`` variant that is Unix only - -.. _start methods: https://docs.python.org/3.8/library/multiprocessing.html#contexts-and-start-methods - - -``trio`` -++++++++ -The ``trio`` backend offers a lightweight async wrapper around ``subprocess`` from the standard library and takes advantage of the ``trio.`` `open_process`_ API. - -.. _open_process: https://trio.readthedocs.io/en/stable/reference-io.html#spawning-subprocesses - - -``multiprocessing`` -+++++++++++++++++++ -There is support for the stdlib's ``multiprocessing`` `start methods`_. -Note that on Windows *spawn* it the only supported method and on \*nix -systems *forkserver* is the best method for speed but has the caveat -that it will break easily (hangs due to broken pipes) if spawning actors -using nested nurseries. - -In general, the ``multiprocessing`` backend **has not proven reliable** -for handling errors from actors more then 2 nurseries *deep* (see `#89`_). -If you for some reason need this consider sticking with alternative -backends. - -.. _#89: https://github.com/goodboy/tractor/issues/89 - -.. _windowsgotchas: - -Windows "gotchas" -^^^^^^^^^^^^^^^^^ -On Windows (which requires the use of the stdlib's `multiprocessing` -package) there are some gotchas. Namely, the need for calling -`freeze_support()`_ inside the ``__main__`` context. Additionally you -may need place you `tractor` program entry point in a seperate -`__main__.py` module in your package in order to avoid an error like the -following :: - - Traceback (most recent call last): - File "C:\ProgramData\Miniconda3\envs\tractor19030601\lib\site-packages\tractor\_actor.py", line 234, in _get_rpc_func - return getattr(self._mods[ns], funcname) - KeyError: '__mp_main__' - - -To avoid this, the following is the **only code** that should be in your -main python module of the program: - -.. code:: python - - # application/__main__.py - import trio - import tractor - import multiprocessing - from . import tractor_app - - if __name__ == '__main__': - multiprocessing.freeze_support() - trio.run(tractor_app.main) - -And execute as:: - - python -m application - - -As an example we use the following code to test all documented examples -in the test suite on windows: - -.. literalinclude:: ../examples/__main__.py - -See `#61`_ and `#79`_ for further details. - -.. _freeze_support(): https://docs.python.org/3/library/multiprocessing.html#multiprocessing.freeze_support -.. _#61: https://github.com/goodboy/tractor/pull/61#issuecomment-470053512 -.. _#79: https://github.com/goodboy/tractor/pull/79 - - -Enabling logging -**************** -Considering how complicated distributed software can become it helps to know -what exactly it's doing (even at the lowest levels). Luckily ``tractor`` has -tons of logging throughout the core. ``tractor`` isn't opinionated on -how you use this information and users are expected to consume log messages in -whichever way is appropriate for the system at hand. That being said, when hacking -on ``tractor`` there is a prettified console formatter which you can enable to -see what the heck is going on. Just put the following somewhere in your code: - -.. code:: python - - from tractor.log import get_console_log - log = get_console_log('trace') + The curated public surface; everything + importable from ``tractor``. +Features +-------- +- **It's just a** ``trio`` **API** — same nursery + discipline, same cancellation semantics, one level + up the process tree. +- *Infinitely nestable* process trees: sub-actors can + spawn sub-actors, supervision stays transitive. +- A "native UX" **multi-process debugger REPL**: + built on pdbp_ with tree-wide tty locking (see + :doc:`guide/debugging`). +- Built-in, cancellation-safe **bidirectional + streaming** via a `cheap or nasty`_ `(un)protocol`_. +- **Typed IPC**: `msgspec`_-backed wire msgs with + optional per-dialog payload specs + (:doc:`guide/msging`). +- Swappable process-spawn backends + modular IPC + transports (TCP today, UDS on same-host, more + planned). +- Optionally distributed_: the same APIs work over + multiple hosts as on multiple cores. +- "**Infected** ``asyncio``" mode: SC-supervise + ``asyncio`` tasks from ``trio`` + (:doc:`guide/asyncio`). +- ``trio`` extension goodies via ``tractor.trionics`` + (acm gathering, single-resource caching, broadcast + channels). + +Where do i start!? +------------------ +The first step to grok ``tractor`` is to get an +intermediate knowledge of ``trio`` and **structured +concurrency** B) + +Some great places to start are, + +- the seminal `blog post`_, +- obviously the `trio docs`_, +- wikipedia's nascent SC_ page, +- the fancy diagrams @ libdill-docs_, + +then come back and hit :doc:`start/quickstart`. -What the future holds ---------------------- -Stuff I'd like to see ``tractor`` do real soon: +.. toctree:: + :hidden: + :maxdepth: 2 -- TLS_, duh. -- erlang-like supervisors_ -- native support for `nanomsg`_ as a channel transport -- native `gossip protocol`_ support for service discovery and arbiter election -- a distributed log ledger for tracking cluster behaviour -- a slick multi-process aware debugger much like in celery_ - but with better `pdb++`_ support -- an extensive `chaos engineering`_ test suite -- support for reactive programming primitives and native support for asyncitertools_ like libs -- introduction of a `capability-based security`_ model + Get started + Big ideas + Guides + API + Project -.. _TLS: https://trio.readthedocs.io/en/latest/reference-io.html#ssl-tls-support -.. _supervisors: https://github.com/goodboy/tractor/issues/22 -.. _nanomsg: https://nanomsg.github.io/nng/index.html -.. _gossip protocol: https://en.wikipedia.org/wiki/Gossip_protocol -.. _celery: http://docs.celeryproject.org/en/latest/userguide/debugging.html -.. _asyncitertools: https://github.com/vodik/asyncitertools -.. _pdb++: https://github.com/antocuni/pdb -.. _capability-based security: https://en.wikipedia.org/wiki/Capability-based_security +.. _trio: https://github.com/python-trio/trio +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _SC: https://en.wikipedia.org/wiki/Structured_concurrency +.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ +.. _trio docs: https://trio.readthedocs.io/en/latest/ +.. _libdill-docs: https://sustrik.github.io/libdill/structured-concurrency.html +.. _pdbp: https://github.com/mdmintz/pdbp +.. _msgspec: https://jcristharif.com/msgspec/ +.. _cheap or nasty: https://zguide.zeromq.org/docs/chapter7/#The-Cheap-or-Nasty-Pattern +.. _(un)protocol: https://zguide.zeromq.org/docs/chapter7/#Unprotocols +.. _distributed: https://en.wikipedia.org/wiki/Distributed_computing diff --git a/docs/project/changelog.rst b/docs/project/changelog.rst new file mode 100644 index 000000000..bb01c125b --- /dev/null +++ b/docs/project/changelog.rst @@ -0,0 +1 @@ +.. include:: ../../NEWS.rst diff --git a/docs/project/dev-tips.rst b/docs/project/dev-tips.rst new file mode 100644 index 000000000..531a28fc2 --- /dev/null +++ b/docs/project/dev-tips.rst @@ -0,0 +1,133 @@ +Hot tips for ``tractor`` hackers +================================ + +This is a (perpetually WIP) guide for newcomers to the project, +mostly to do with dev, testing, CI and release gotchas, reminders +and best practises. + +``tractor`` is a fairly novel project compared to most since it is +effectively a new way of doing distributed computing in Python and +is much closer to working with an "application level runtime" +(like erlang OTP or scala's akka project) than it is a traditional +Python library. As such, having an arsenal of tools and recipes +for figuring out the right way to debug problems when they do +arise is somewhat of a necessity. + +Making a release +---------------- + +Nothing fancy: the traditional PyPA flow on the hatchling_ build +backend, with uv_ doing the driving and towncrier_ generating the +changelog. + +1. collect news fragments: user-facing changes should land with a + small ``.rst`` snippet under ``nooz/`` (see ``nooz/HOWTO.rst``; + fragment types are ``feature``, ``bugfix``, ``doc`` and + ``trivial``), + +2. render them into ``NEWS.rst``:: + + uvx towncrier build --version + +3. build and upload (testpypi first if you're being careful):: + + uv build + uvx twine upload -r testpypi dist/* + uvx twine upload dist/* + +How you organize built artifacts under ``dist/`` locally (per +release sub-dirs and such) is entirely up to you. + +Keep in mind that PyPi releases tend to lag the ``main`` branch +since we develop in the open — ``main`` is usually the thing to +run when you want the latest. + +Debugging and monitoring actor trees +------------------------------------ + +Your "what is my tree doing right now?" toolbox, in escalation +order: + +**Live process-tree view** — keep a ``watch``-ed ``pstree`` +running in a side terminal; actor procs are recognizable by their +``_subactor[@]`` process titles. The exact +one-liners (plus the ``pgrep`` marker recipes) live in +:doc:`/guide/testing`. + +**SIGUSR1 task-tree dumps** — boot any tree with +``enable_stack_on_sig=True`` (or export +``TRACTOR_ENABLE_STACKSCOPE=1``) and every actor installs a +stackscope_ signal handler. Then from any shell:: + + # dump every actor's live trio task tree: + pkill --signal SIGUSR1 -f + + # or for a single process: + kill -SIGUSR1 $(pgrep -f ) + +Each dump is also tee'd (append-mode) to +``/tmp/tractor-stackscope-.log`` so you still get output +under pytest capture or in CI. This works *without* debug-mode +being enabled — it's the lightest-weight hang-investigation tool +in the box. + +**The built-in multi-process debugger** — ``debug_mode=True`` +plus :func:`tractor.pause` and friends: the heavyweight champ for +interactive, REPL-driven inspection of a whole tree (including +crash handling). Remember pytest capture interplay — see +:doc:`/guide/testing`. + +**Post-mortem zombie sweeps** — ``scripts/tractor-reap`` for the +(should-be-rare!) cases where hacking on the runtime itself wedges +a tree: a SIGINT-first, structured concurrency (SC) polite +escalation, plus ``--shm`` and ``--uds`` leaked-resource sweeps. + +Using the log system to trace ``trio`` task flow +------------------------------------------------ + +The logging system is oriented around the **stack "layers" of the +runtime**, letting you trace logical abstraction layers in the +code — errors, cancellation, IPC and streaming, the low level +transport and wire protocols — independently of one another. + +Concretely, ``tractor.log.get_logger()`` returns a +``StackLevelAdapter`` sporting extra level-methods beyond the +stdlib set, including: + +- ``.cancel()`` — cancellation-machinery flow, + +- ``.runtime()`` — actor-runtime lifecycle chatter, + +- ``.devx()`` — debugger/devx tooling internals, + +- ``.transport()`` — wire-level msging events. + +To get console output at any level from your own code:: + + from tractor.log import get_console_log + get_console_log('cancel') + +or, runtime-wide without touching code, just export +``TRACTOR_LOGLEVEL=cancel`` (the env-var wins over caller-passed +levels; great for test runs). + +When you want only *one subsystem* cranked, the suite's ``--tl`` +flag (and ``tractor.log.apply_logspec()``) accept a per-sublogger +spec:: + + uv run pytest tests/... --tl 'devx:runtime,trionics:cancel' + +Every record's header includes the emitting actor and task names, +so cross-process flows can be stitched back together by eyeball +(or grep). + +.. seealso:: + - :doc:`/guide/testing` — running the suite, watching trees + live, examples-as-tests conventions and the zombie-reaper. + - :doc:`/guide/discovery` — the registrar mechanics you'll + bump into when running multiple trees on one host. + +.. _hatchling: https://hatch.pypa.io/latest/ +.. _uv: https://docs.astral.sh/uv/ +.. _towncrier: https://towncrier.readthedocs.io/en/stable/ +.. _stackscope: https://github.com/oremanj/stackscope diff --git a/docs/project/index.rst b/docs/project/index.rst new file mode 100644 index 000000000..07fc1f7d2 --- /dev/null +++ b/docs/project/index.rst @@ -0,0 +1,60 @@ +Project +======= +Everything meta: where the project's been (the +:doc:`changelog`), where it's going (the roadmap below), +how to hack on it (:doc:`dev-tips` and +:doc:`/guide/testing`) and where to find the humans. + +.. toctree:: + :maxdepth: 1 + + changelog + dev-tips + +What the future holds +--------------------- +Help us push toward the future of distributed Python! +Planned (or dreamed of) work non-comprehensively +includes, + +- Erlang-style supervisors via composed context + managers (see `#22`_), +- typed capability-based (dialog) protocols, ie. + evolving our `msg-spec`_ system into per-dialog + contracts (see `#196`_ with draft work in `#311`_), +- a higher level "service manager" API for daemon + lifetime mgmt over actor trees (in the works on an + experimental branch as ``tractor.hilevel``), +- richer `discovery`_ via gossip and/or + `rendezvous protocol`_ approaches (today's registrar + is intentionally naive), +- more IPC transports: the current ``tcp`` | ``uds`` + pair wants friends (QUIC, shm-ring-buffers, RUDP, + wireguard tunnels), +- an extensive `chaos engineering`_ test suite, +- a respawn-from-REPL system for crashed (sub-)actors. + +Feel like saying hi? +-------------------- +This project is very much coupled to the ongoing +development of ``trio`` (i.e. ``tractor`` gets most of +its ideas from that brilliant community). If you want +to help, have suggestions or just want to say hi, +please feel free to reach us in our `matrix channel`_. +If matrix seems too hip, we're also mostly all in the +`trio gitter channel`_! + +Contributions of all kinds welcome: docs, examples, +bug reports, new transports, supervisor strategies, +philosophical debates about what an "actor model" +really is B) + +.. _#22: https://github.com/goodboy/tractor/issues/22 +.. _#196: https://github.com/goodboy/tractor/issues/196 +.. _#311: https://github.com/goodboy/tractor/pull/311 +.. _msg-spec: https://jcristharif.com/msgspec/ +.. _discovery: https://zguide.zeromq.org/docs/chapter8/#Discovery +.. _rendezvous protocol: https://en.wikipedia.org/wiki/Rendezvous_protocol +.. _chaos engineering: https://principlesofchaos.org/ +.. _matrix channel: https://matrix.to/#/!tractor:matrix.org +.. _trio gitter channel: https://gitter.im/python-trio/general diff --git a/docs/start/index.rst b/docs/start/index.rst new file mode 100644 index 000000000..9e8682706 --- /dev/null +++ b/docs/start/index.rst @@ -0,0 +1,36 @@ +Getting started +=============== +Welcome aboard B) + +Real talk before any code: the first step to grok ``tractor`` is +to get an intermediate knowledge of ``trio`` and **structured +concurrency** (SC). ``tractor`` **is just** ``trio`` - but with +nurseries for process management and cancel-able streaming IPC - +so every rule you already know about task lifetimes, cancellation +and error propagation keeps holding, just now across process (and +host!) boundaries. Some great places to start are, + +- the seminal `blog post`_, +- obviously the `trio docs`_, +- wikipedia's nascent SC_ page, +- the fancy diagrams @ libdill-docs_. + +Once you've taken in (some of) the canon, get installed and go +spawn your first actor tree: + +.. toctree:: + :maxdepth: 1 + + install + quickstart + +.. seealso:: + + Already installed and itching? Jump straight to + :doc:`/start/quickstart`; once you're through the on-ramp the + :doc:`guide pages ` take each subsystem deeper. + +.. _blog post: https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/ +.. _trio docs: https://trio.readthedocs.io/en/latest/ +.. _SC: https://en.wikipedia.org/wiki/Structured_concurrency +.. _libdill-docs: https://sustrik.github.io/libdill/structured-concurrency.html diff --git a/docs/start/install.rst b/docs/start/install.rst new file mode 100644 index 000000000..7fc0a852d --- /dev/null +++ b/docs/start/install.rst @@ -0,0 +1,75 @@ +Install +======= +``tractor`` is still in an *alpha-near-beta-stage* for many of +its subsystems, however we are very close to having a stable +lowlevel runtime and API. Expect the occasional rough edge (and +feel free to report it!). + +Supported platforms +------------------- +- **python**: ``>=3.13,<3.15`` - yes, we ride near the front of + the release train; 3.14 additionally unlocks the experimental + PEP-734 sub-interpreter spawn backend, +- **linux**: the primary development and CI platform, +- **macos**: officially supported, +- **windows**: currently *untested* - we disabled CI-testing on + windows a while back and haven't had the cycles to revive it. + It mostly worked historically; if you're a windows person we'd + love a hand getting it back in the test matrix. + +With ``uv`` (preferred) +----------------------- +We use the very hip uv_ for project mgmt and recommend you do +too. Add ``tractor`` to your project:: + + uv add tractor + +or, since the git ``main`` branch is often much further ahead +than any latest release (see the PyPi note below), track ``main`` +directly:: + + uv add "tractor @ git+https://github.com/goodboy/tractor.git" + +From PyPi +--------- +We ofc also offer "releases" on PyPi_:: + + pip install tractor + +Just note that **YMMV** since ``main`` is usually well ahead of +the latest published alpha; when in doubt go with a git install +as per above (or hack from source as per below). + +From source +----------- +To run the bundled examples in-tree, or to start hacking on the +code base, clone and sync a dev env:: + + git clone https://github.com/goodboy/tractor.git + cd tractor + uv sync --dev + uv run python examples/rpc_bidir_streaming.py + +Consider activating a virtual/project-env before starting to hack +on the code base:: + + # you could use plain ol' venvs + # https://docs.astral.sh/uv/pip/environments/ + uv venv tractor_py313 --python 3.13 + + # but @goodboy prefers the more explicit (and shell agnostic) + # https://docs.astral.sh/uv/configuration/environment/#uv_project_environment + UV_PROJECT_ENVIRONMENT="tractor_py313" + + # hint hint, enter @goodboy's fave shell B) + uv run --dev xonsh + +.. seealso:: + + All set? Go boot a process tree in :doc:`/start/quickstart`. + And if the install fights you, swing by the `matrix channel`_ + and we'll sort it out. + +.. _uv: https://docs.astral.sh/uv/ +.. _PyPi: https://pypi.org/project/tractor/ +.. _matrix channel: https://matrix.to/#/!tractor:matrix.org diff --git a/docs/start/quickstart.rst b/docs/start/quickstart.rst new file mode 100644 index 000000000..75bbafb24 --- /dev/null +++ b/docs/start/quickstart.rst @@ -0,0 +1,264 @@ +Quickstart +========== +Time to spawn something B) + +If you take one thing from this page make it this: ``tractor`` +**is just** ``trio`` - but with nurseries for process management +and cancel-able streaming IPC. Every "*actor*" you'll meet below +is a plain Python **process** running its own ``trio.run()`` +scheduled task tree, linked back to its parent through an IPC +protocol which keeps the whole tree `structured concurrency`_ +(SC) compliant end-to-end. If you know your nursery_ semantics +you already know most of ``tractor``; we just stretch them across +the process boundary. + +.. d2:: diagrams/actor_tree.d2 + :alt: a supervision tree of actor processes + :margin: + :caption: every arrow is a parent which **must wait** on its kids + +Your first actor tree +--------------------- +``trio`` takes the hard-line position that a parent task **must +wait** on the children it spawns; causality_ is paramount! So +does ``tractor``, one abstraction layer up: +``tractor.open_nursery()`` yields an ``ActorNursery`` which +**must** wait on its spawned *subactors* to complete (or error) +before the ``async with`` block exits, in the same causal_ way a +``trio`` nursery waits on its subtasks. That includes any one +child's crash cancelling all of its siblings: *one-cancels-all* +supervision, `exactly like trio`_. + +Enough preamble, spawn a process: + +.. literalinclude:: ../../examples/actor_spawning_and_causality.py + :caption: examples/actor_spawning_and_causality.py + :language: python + +Run it:: + + $ python examples/actor_spawning_and_causality.py + Dang that's beautiful + +What's going on here? + +- ``trio.run(main)`` starts the **root actor**; the ``tractor`` + runtime boots *implicitly* inside ``tractor.open_nursery()`` + whenever it isn't already up. No special entrypoint, no + framework takeover - it's just a ``trio`` app, +- inside ``main()`` a *subactor* is spawned via + ``ActorNursery.run_in_actor()`` and told to run exactly one + function: ``cellar_door()``, +- you get back a ``Portal``: your handle for invoking tasks in + the new process's (separate!) memory domain. We lean on it + much harder in the next section, +- the subactor, *some_linguist*, boots a fresh ``trio.run()`` in + a **new process** and executes ``cellar_door()`` as its *main + task* (note the child proving it is *not* the root with + ``tractor.is_root_process()``), then ships the return value + back over IPC, +- the parent grabs that *final result* with + ``await portal.wait_for_result()``, much like you'd expect + from a "future" - except causality is preserved: the nursery + block only exits once the child is *done*, dead, and reaped. + +.. margin:: Just need a worker pool? + + If all you want is to throw *sync* functions at your cores, + also check out trio-parallel_. ``tractor`` is aimed at + structured, (possibly) distributed *trees* of cooperating + ``trio`` programs; a worker pool is a trivial special case. + +.. note:: + + ``run_in_actor()`` is the *convenience* wrapper: one-shot + spawn-run-reap semantics for when a subactor's entire job is + a single function call. The core primitives are + ``ActorNursery.start_actor()`` (next up) paired with + ``Portal.open_context()`` for full, SC-linked cross-actor + dialogs - see :doc:`/guide/context`. + +Daemon actors and RPC +--------------------- +A ``run_in_actor()``-spawned actor terminates when its main task +returns. But often you want long-lived *daemon* actors instead: +spawned once, then serving (allowlisted) RPC requests until told +otherwise. That's ``start_actor()``: + +.. literalinclude:: ../../examples/actor_spawning_and_causality_with_daemon.py + :caption: examples/actor_spawning_and_causality_with_daemon.py + :language: python + +Two lifetime rules to internalize: + +- a ``run_in_actor()`` actor lives exactly as long as its main + task; the nursery waits for that function (and thus the + process) to complete before unblocking, +- a ``start_actor()`` actor *lives forever* - an RPC daemon the + nursery will happily wait on **indefinitely** - until some + task explicitly cancels it via ``Portal.cancel_actor()`` (as + above), or its parent nursery is cancelled wholesale. + +.. tip:: + + Want your *entire program* to just be a long-lived RPC + daemon? ``tractor.run_daemon()`` is the blocking shorthand: + it ``trio.run()``\s a root actor which serves requests until + cancelled. + +The ``enable_modules=[__name__]`` kwarg is the other thing to +notice: it lists the module paths the subactor will load and +*expose* for remote invocation. +``await portal.run(movie_theatre_question)`` works because this +very module is in that allowlist (and note we call it twice; the +daemon happily serves repeat requests). Ask for a function from +any module *not* enabled and you're denied with a +``ModuleNotExposed`` error: a simple, capability-style +restriction mechanism built on Python's own module system. + +We are *processes* +------------------ +Why processes (and not, say, threads)? Python has a GIL and an +`actor model`_ by definition shares **nothing** between its +concurrent units, so real OS processes are the natural fit: you +get all your cores locally, and since actors only ever talk via +IPC, the exact same code distributes over multiple hosts without +modification. + +Of course, the moment you hear "process trees" you should be +asking: *what about zombies?* Watch ``tractor`` eat one for +breakfast - run this while monitoring your process tree:: + + $TERM -e watch -n 0.1 "pstree -a $$" \ + & python examples/parallelism/we_are_processes.py \ + && kill $! + +.. literalinclude:: ../../examples/parallelism/we_are_processes.py + :caption: examples/parallelism/we_are_processes.py + :language: python + +.. margin:: Who's who in ``pstree``? + + Every subactor (best-effort, via the optional + ``setproctitle`` dep) re-titles its OS process like + ``_subactor[worker_0@]``, so ``pstree``/``htop``/ + ``pgrep -f`` can tell your actors apart at a glance. + +You'll see something like:: + + $ python examples/parallelism/we_are_processes.py + Yo, i'm 'worker_2' running in pid 1777246 + Yo, i'm 'worker_0' running in pid 1777244 + Yo, i'm 'worker_3' running in pid 1777247 + Yo, i'm 'worker_1' running in pid 1777245 + This process tree will self-destruct in 1 sec... + Zombies Contained + +(The worker lines land in whatever order the OS schedules them; +they're separate *processes*, racing, and that's the point.) + +An actor is spawned per core, each parks itself in +``trio.sleep_forever()``... and then the root *crashes on +purpose*. The ``ActorNursery`` responds with hard ``trio`` +discipline: every child is cancelled, every process is reaped, +the error propagates to ``trio.run()``, and your terminal prints +``Zombies Contained``. No orphans, no ``kill -9`` archaeology in +``htop`` afterwards. + +.. note:: + + **The zombie-safety guarantee**: ``tractor`` tries to protect + you from zombies, *no matter what*. If you can create zombie + child processes (without using a system signal) it **is a + bug** - please report it so we can hunt it down. + +A trynamic first scene +---------------------- +So far the root actor has done all the talking, but subactors +can just as well discover and call *each other*. Let's direct a +couple actors and have them run their lines for the hip new film +we're shooting: + +.. literalinclude:: ../../examples/a_trynamic_first_scene.py + :caption: examples/a_trynamic_first_scene.py + :language: python + +The script of the scene (runtime ``INFO`` log lines trimmed):: + + $ python examples/a_trynamic_first_scene.py + Alright... Action! + Hi my name is gretchen + Hi my name is donny + CUTTTT CUUTT CUT!!! Donny!! You're supposed to say... + +The new tricks in play: + +- two subactors, *donny* and *gretchen*, are each told to run + ``say_hello()`` targeting the *other* by name, +- ``tractor.wait_for_actor()`` blocks until the named peer has + registered with the tree's *registrar* (every actor announces + itself at boot), then yields a ``Portal`` connected + **directly** to that peer, +- each actor invokes its partner's ``hi()`` over that portal: + actor-to-actor RPC with the root merely *directing* - and both + final lines flow back to ``main()`` via + ``await portal.wait_for_result()``, +- ``tractor.log.get_console_log("INFO")`` cranks up runtime + logging so you can watch the spawn/register/cancel machinery + narrate itself; remove it for a quiet set. + +Cross-actor calls look just like (async) function calls; there +are no proxy objects and no shared references, only messages B) + +Crash handling, native feeling +------------------------------ +One last teaser before the guide proper. Flip exactly one +switch: + +.. code:: python + + async with tractor.open_nursery( + debug_mode=True, + ) as an: + ... + +and any crash, in *any* actor at *any* depth of the tree, drops +your terminal into a multi-process-safe pdbp_ REPL at the +offending frame, with the rest of the tree held back from +clobbering the tty. ``await tractor.pause()`` likewise gives you +a breakpoint that *just works* inside subprocesses. We think it +might be the first native multi-process debugging UX for Python; +get the full tour in :doc:`/guide/debugging`. + +Where to next? +-------------- +You can now boot a runtime, spawn one-shot and daemon actors, +make cross-process RPC calls, and contain zombies: that's the +on-ramp done. The guide takes each subsystem deeper, + +- :doc:`/explain/sc-distributed` - the structured concurrency + worldview and how ``tractor`` extends it across processes, +- :doc:`/guide/spawning` - everything ``ActorNursery``: spawn + kwargs, lifetimes and supervision semantics, +- :doc:`/guide/rpc` - the ``Portal`` in depth: calling into + another actor's memory domain, +- :doc:`/guide/context` - the core API: ``@tractor.context`` + endpoints, the ``ctx.started()`` handshake, and SC-linked + cross-actor task pairs, +- :doc:`/guide/streaming` - bidirectional ``MsgStream`` dialogs + and fan-out broadcasting, +- :doc:`/guide/debugging` - the multi-process REPL, crash + handling mode, and ``tractor.pause()``, +- :doc:`/guide/asyncio` - "infected ``asyncio``" mode: SC + supervision wrapped around ``asyncio`` tasks, +- :doc:`/guide/discovery` - registries, service daemons, and + finding actors from anywhere in (or out of) the tree. + +.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency +.. _nursery: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning +.. _causality: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#c-c-c-c-causality-breaker +.. _causal: https://vorpus.org/blog/some-thoughts-on-asynchronous-api-design-in-a-post-asyncawait-world/#causality +.. _exactly like trio: https://trio.readthedocs.io/en/latest/reference-core.html#cancellation-semantics +.. _actor model: https://en.wikipedia.org/wiki/Actor_model +.. _trio-parallel: https://github.com/richardsheridan/trio-parallel +.. _pdbp: https://github.com/mdmintz/pdbp diff --git a/examples/a_trynamic_first_scene.py b/examples/a_trynamic_first_scene.py index 1d531255e..05d61ba9a 100644 --- a/examples/a_trynamic_first_scene.py +++ b/examples/a_trynamic_first_scene.py @@ -35,8 +35,8 @@ async def main(): name='gretchen', other_actor='donny', ) - print(await gretchen.result()) - print(await donny.result()) + print(await gretchen.wait_for_result()) + print(await donny.wait_for_result()) print("CUTTTT CUUTT CUT!!! Donny!! You're supposed to say...") diff --git a/examples/actor_spawning_and_causality.py b/examples/actor_spawning_and_causality.py index 119726dc0..2232ae3ea 100644 --- a/examples/actor_spawning_and_causality.py +++ b/examples/actor_spawning_and_causality.py @@ -20,7 +20,7 @@ async def main(): # The ``async with`` will unblock here since the 'some_linguist' # actor has completed its main task ``cellar_door``. - print(await portal.result()) + print(await portal.wait_for_result()) if __name__ == '__main__': diff --git a/examples/nested_actor_tree.py b/examples/nested_actor_tree.py new file mode 100644 index 000000000..a0c66b88b --- /dev/null +++ b/examples/nested_actor_tree.py @@ -0,0 +1,105 @@ +''' +Demonstrate a (3-level) nested actor tree where one RPC from +the root fans out through a mid-tier 'supervisor' actor to +2 'leaf' worker actors and an aggregate result is relayed +back up. + +The process tree should look approximately like: + +python examples/nested_actor_tree.py +`-python -m tractor._child --uid ('supervisor', '7c9b1039 ..) + |-python -m tractor._child --uid ('leaf_1', '92d62f50 ..) + `-python -m tractor._child --uid ('leaf_2', 'de91fdf5 ..) + +Teardown runs inside-out: the supervisor cancels its leaves +first, then the root cancels the supervisor; watch the +prints to see the ordering. + +''' +import trio +import tractor + + +async def compute_square(x: int) -> int: + ''' + Tiny "work unit" run inside a leaf actor. + + ''' + name: str = tractor.current_actor().name + print(f'{name}: squaring {x}') + return x * x + + +@tractor.context +async def fan_out_squares( + ctx: tractor.Context, + vals: list[int], +) -> list[int]: + ''' + Spawn a (nested) pair of leaf actors, fan the input vals + out across them round-robin style, then return the + aggregated squares to our parent. + + ''' + async with tractor.open_nursery() as an: + portals: list[tractor.Portal] = [] + for i in (1, 2): + portals.append( + await an.start_actor( + f'leaf_{i}', + enable_modules=[__name__], + ) + ) + # unblock the parent's `.open_context()` entry and + # report which leaves came up. + await ctx.started( + [p.chan.aid.name for p in portals] + ) + squares: dict[int, int] = {} + + async def run_in_leaf( + portal: tractor.Portal, + x: int, + ) -> None: + squares[x] = await portal.run( + compute_square, + x=x, + ) + + # fan out one sub-RPC per input val, concurrently. + async with trio.open_nursery() as tn: + for i, x in enumerate(vals): + tn.start_soon( + run_in_leaf, + portals[i % len(portals)], + x, + ) + # graceful inside-out teardown: leaves go first! + for portal in portals: + leaf_name: str = portal.chan.aid.name + print(f'supervisor: cancelling {leaf_name}') + await portal.cancel_actor() + return [squares[x] for x in vals] + + +async def main() -> None: + async with tractor.open_nursery() as an: + portal = await an.start_actor( + 'supervisor', + enable_modules=[__name__], + ) + async with portal.open_context( + fan_out_squares, + vals=[1, 2, 3, 4], + ) as (ctx, leaf_names): + print(f'root: supervisor spawned {leaf_names}') + squares: list[int] = await ctx.wait_for_result() + assert squares == [1, 4, 9, 16] + print(f'root: aggregate result {squares}') + print('root: cancelling supervisor') + await portal.cancel_actor() + print('root: tree torn down, what zombies?') + + +if __name__ == '__main__': + trio.run(main) diff --git a/examples/parallelism/_concurrent_futures_primes.py b/examples/parallelism/concurrent_futures_primes.py similarity index 61% rename from examples/parallelism/_concurrent_futures_primes.py rename to examples/parallelism/concurrent_futures_primes.py index 7246b8649..1ce5ee2a8 100644 --- a/examples/parallelism/_concurrent_futures_primes.py +++ b/examples/parallelism/concurrent_futures_primes.py @@ -1,7 +1,19 @@ +''' +The pure-stdlib `concurrent.futures.ProcessPoolExecutor` +primes demo (from the std docs) verbatim; the baseline twin +of `concurrent_actors_primes.py`. + +The `async def main()` + `trio.run()` shim at the bottom only +exists so the docs-example test runner can exercise this +script; the executor code itself is untouched stdlib fare. + +''' import time import concurrent.futures import math +import trio + PRIMES = [ 112272535095293, 112582705942171, @@ -26,7 +38,7 @@ def is_prime(n): return True -def main(): +def check_primes(): with concurrent.futures.ProcessPoolExecutor() as executor: start = time.time() @@ -36,8 +48,14 @@ def main(): print(f'processing took {time.time() - start} seconds') +async def main() -> None: + # thin shim: the pool blocks this (sole) trio task + # which is just fine for a one-shot baseline script. + check_primes() + + if __name__ == '__main__': start = time.time() - main() + trio.run(main) print(f'script took {time.time() - start} seconds') diff --git a/examples/parallelism/single_func.py b/examples/parallelism/single_func.py index 8118b024e..c4ea29e37 100644 --- a/examples/parallelism/single_func.py +++ b/examples/parallelism/single_func.py @@ -33,7 +33,7 @@ async def main(): await burn_cpu() # wait on result from target function - pid = await portal.result() + pid = await portal.wait_for_result() # end of nursery block print(f"Collected subproc {pid}") diff --git a/examples/service_daemon_discovery.py b/examples/service_daemon_discovery.py new file mode 100644 index 000000000..a9086b8c3 --- /dev/null +++ b/examples/service_daemon_discovery.py @@ -0,0 +1,68 @@ +''' +Demonstrate the "service daemon" pattern: a named, +long-lived actor spawned via `ActorNursery.start_actor()` +which any other task can locate through the registrar using +`tractor.find_actor()` / `tractor.wait_for_actor()` - no +spawn-portal required - and RPC into directly. + +Teardown is explicit and graceful via `portal.cancel_actor()` +once the clients are done. + +''' +import trio +import tractor + +_quotes: dict[str, float] = { + 'btcusdt': 66_000.5, + 'ethusdt': 3_500.25, +} + + +async def get_quote(sym: str) -> float: + ''' + Look up the "current" quote for a symbol. + + ''' + name: str = tractor.current_actor().name + print(f'{name}: serving quote for {sym!r}') + return _quotes[sym] + + +async def client_task() -> None: + ''' + Locate the quote service by name and RPC it; note no + spawn-nursery/portal reference is ever passed in here! + + ''' + # a lookup miss yields `None` (not an error). + async with tractor.find_actor('no_such_svc') as portal: + assert portal is None + print('client: "no_such_svc" is not registered') + # block until the service shows up in the registry, + # then call into it through the delivered portal. + async with tractor.wait_for_actor('quote_svc') as portal: + quote: float = await portal.run( + get_quote, + sym='btcusdt', + ) + print(f'client: got btcusdt quote {quote}') + + +async def main() -> None: + async with tractor.open_nursery() as an: + portal = await an.start_actor( + 'quote_svc', + enable_modules=[__name__], + ) + # run the client in a separate task which discovers + # the daemon purely by its registered name. + async with trio.open_nursery() as tn: + tn.start_soon(client_task) + # explicit graceful teardown of the daemon. + print('root: cancelling quote_svc') + await portal.cancel_actor() + print('root: service shut down cleanly') + + +if __name__ == '__main__': + trio.run(main) diff --git a/examples/streaming_broadcast_fanout.py b/examples/streaming_broadcast_fanout.py new file mode 100644 index 000000000..9f6900cef --- /dev/null +++ b/examples/streaming_broadcast_fanout.py @@ -0,0 +1,84 @@ +''' +Demonstrate fanning out ONE inter-actor `MsgStream` to N +local (parent-side) trio tasks using `MsgStream.subscribe()`: +each subscriber gets its own `BroadcastReceiver` copy of +every msg from the single underlying IPC stream. + +The child waits for a 'go' msg so that all subscribers are +guaranteed-attached before the first tick is sent; when the +child's stream closes each subscriber's `async for` ends +cleanly. + +''' +import trio +import tractor + + +@tractor.context +async def tick_stream( + ctx: tractor.Context, + count: int, +) -> None: + ''' + Send `count` "ticks" once the parent says go. + + ''' + await ctx.started(count) + async with ctx.open_stream() as stream: + # wait for the go-signal ensuring every parent-side + # subscriber is attached before any tick is sent. + assert await stream.receive() == 'go' + for i in range(count): + await stream.send(i) + # falling out gracefully closes our stream side; + # all parent-side subscribers see end-of-channel. + + +async def consume( + name: str, + stream: tractor.MsgStream, + task_status: trio.TaskStatus = trio.TASK_STATUS_IGNORED, +) -> None: + ''' + Consume a private broadcast-copy of the IPC stream. + + ''' + async with stream.subscribe() as bcaster: + task_status.started() + ticks: list[int] = [] + async for tick in bcaster: + print(f'{name}: rx {tick}') + ticks.append(tick) + # EVERY subscriber gets its own full copy B) + print(f'{name}: stream ended, got {ticks}') + + +async def main() -> None: + async with tractor.open_nursery() as an: + portal = await an.start_actor( + 'ticker', + enable_modules=[__name__], + ) + async with ( + portal.open_context( + tick_stream, + count=5, + ) as (ctx, first), + ctx.open_stream() as stream, + ): + assert first == 5 + async with trio.open_nursery() as tn: + # use `.start()` so each consumer is known + # to be subscribed before the ticks flow. + for i in range(3): + await tn.start( + consume, + f'sub_{i}', + stream, + ) + await stream.send('go') + await portal.cancel_actor() + + +if __name__ == '__main__': + trio.run(main) diff --git a/examples/typed_payloads.py b/examples/typed_payloads.py new file mode 100644 index 000000000..c6352de59 --- /dev/null +++ b/examples/typed_payloads.py @@ -0,0 +1,85 @@ +''' +Demonstrate "typed messaging": applying a `msgspec.Struct` +payload-type-spec to an IPC context via +`@tractor.context(pld_spec=...)`. + +The child's `ctx.started()` value is stringently (round-trip) +validated against the spec *on the send side*, so a mistyped +payload raises a `tractor.MsgTypeError` before it ever hits +the wire; stream payloads are checked on `receive()` and +decode natively to the struct type. + +''' +from msgspec import Struct + +import trio +import tractor + + +class Point(Struct): + ''' + A simple 2D-coordinate msg-payload type. + + ''' + x: int + y: int + + +@tractor.context(pld_spec=Point|None) +async def point_doubler( + ctx: tractor.Context, +) -> None: + ''' + Stream back each received `Point` with doubled fields. + + ''' + # deliberately send a non-`Point` as our started-value; + # the send-side pld-spec validation catches it locally + # BEFORE anything is shipped over IPC. + try: + await ctx.started('this is no Point..') + except tractor.MsgTypeError: + print( + 'child: just as planned, a `str` payload failed ' + 'the `Point|None` pld-spec B)' + ) + # now do it right; the parent receives this as the 2nd + # element of its `.open_context()` entry tuple. + await ctx.started(Point(x=0, y=0)) + async with ctx.open_stream() as stream: + async for pt in stream: + # natively decoded to our struct type! + assert type(pt) is Point + await stream.send( + Point( + x=pt.x * 2, + y=pt.y * 2, + ) + ) + + +async def main() -> None: + async with tractor.open_nursery() as an: + portal = await an.start_actor( + 'point_doubler', + enable_modules=[__name__], + ) + async with ( + portal.open_context( + point_doubler, + ) as (ctx, first), + ctx.open_stream() as stream, + ): + # the (validated) started-value from the child + assert first == Point(x=0, y=0) + for i in range(3): + await stream.send(Point(x=i, y=i)) + doubled: Point = await stream.receive() + assert doubled == Point(x=i * 2, y=i * 2) + print(f'parent: rx {doubled}') + # explicitly teardown the daemon-actor + await portal.cancel_actor() + + +if __name__ == '__main__': + trio.run(main) diff --git a/examples/uds_transport_actor_tree.py b/examples/uds_transport_actor_tree.py new file mode 100644 index 000000000..956dd89e7 --- /dev/null +++ b/examples/uds_transport_actor_tree.py @@ -0,0 +1,53 @@ +''' +Demonstrate an actor tree which talks over unix-domain-socket +(UDS) transport instead of the default TCP: pass +`enable_transports=['uds']` when opening the root and every +subactor inherits the preference. + +The child's channel address is a filesystem socket path (no +TCP port in sight!) and, as a kernel-provided bonus, the +peer's pid is exchanged for free via `SO_PEERCRED`. + +''' +import os + +import trio +import tractor + + +async def report_addr() -> str: + ''' + Return this actor's own accept (bind) addr + pid. + + ''' + actor = tractor.current_actor() + addr: tuple = actor.accept_addr + pid: int = os.getpid() + return f'{actor.name}@{addr} pid={pid}' + + +async def main() -> None: + async with tractor.open_nursery( + enable_transports=['uds'], + ) as an: + portal = await an.start_actor( + 'uds_child', + enable_modules=[__name__], + ) + # the channel's remote addr is a `UDSAddress`: a + # filesystem socket path, NOT a (host, port) pair! + raddr = portal.chan.raddr + assert raddr.proto_key == 'uds' + print( + f'portal chan tpt proto: {raddr.proto_key!r}\n' + f'portal chan sock file: {raddr.sockpath}\n' + f'kernel-reported peer pid: {raddr.maybe_pid}\n' + ) + # ask the child for its own bind addr: also a + # socket-file path under the runtime dir. + print(f'child says: {await portal.run(report_addr)}') + await portal.cancel_actor() + + +if __name__ == '__main__': + trio.run(main) diff --git a/pyproject.toml b/pyproject.toml index 7fc7c2670..16c916e17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -119,12 +119,19 @@ eventfd = [ subints = [ "msgspec>=0.21.0", ] -# TODO, add these with sane versions; were originally in -# `requirements-docs.txt`.. -# docs = [ -# "sphinx>=" -# "sphinx_book_theme>=" -# ] +# docs generation; build locally via, +# uv run --group docs make -C docs html +# diagrams re-render when a `d2` bin is found, see +# `docs/_ext/d2diagrams.py` (eg. via `nix run +# nixpkgs#d2` and the `D2_BIN` env var). +docs = [ + "sphinx>=9.1,<10", + "pydata-sphinx-theme>=0.18,<0.19", + "sphinx-design>=0.7,<0.8", + "sphinx-copybutton>=0.5.2,<0.6", + "sphinxext-opengraph>=0.13,<0.14", + "sphinx-togglebutton>=0.4.5,<0.5", +] # ------ dependency-groups ------ [tool.uv.dependency-groups] diff --git a/tractor/_root.py b/tractor/_root.py index 78a1be56e..a70f1c39c 100644 --- a/tractor/_root.py +++ b/tractor/_root.py @@ -155,7 +155,6 @@ def block_bps(*args, **kwargs): os.environ.pop('PYTHONBREAKPOINT', None) - @acm async def open_root_actor( *, @@ -186,6 +185,7 @@ async def open_root_actor( # enables the multi-process debugger support debug_mode: bool = False, maybe_enable_greenback: bool = False, # `.pause_from_sync()/breakpoint()` support + # ^XXX NOTE^ the perf implications of use, # https://greenback.readthedocs.io/en/latest/principle.html#performance enable_stack_on_sig: bool = False, diff --git a/tractor/runtime/_state.py b/tractor/runtime/_state.py index 11e0c0fd0..1c019c768 100644 --- a/tractor/runtime/_state.py +++ b/tractor/runtime/_state.py @@ -140,7 +140,7 @@ def update( # `debug_mode: bool` settings '_debug_mode': False, # bool 'repl_fixture': False, # |AbstractContextManager[bool] - + 'use_greenback': False, # `.pause_from_sync()`/`breakpoint()` 'use_stackscope': False, # trio-task-stack dumps on SIGUSR1 diff --git a/uv.lock b/uv.lock index d366f1845..f892c5510 100644 --- a/uv.lock +++ b/uv.lock @@ -6,6 +6,27 @@ resolution-markers = [ "python_full_version < '3.14'", ] +[[package]] +name = "accessible-pygments" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c1/bbac6a50d02774f91572938964c582fff4270eee73ab822a4aeea4d8b11b/accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872", size = 1377899, upload-time = "2024-05-10T11:23:10.216Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/3f/95338030883d8c8b91223b4e21744b04d11b161a3ef117295d8241f50ab4/accessible_pygments-0.0.5-py3-none-any.whl", hash = "sha256:88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7", size = 1395903, upload-time = "2024-05-10T11:23:08.421Z" }, +] + +[[package]] +name = "alabaster" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/f8/d9c74d0daf3f742840fd818d69cfae176fa332022fd44e3469487d5a9420/alabaster-1.0.0.tar.gz", hash = "sha256:c00dca57bca26fa62a6d7d0a9fcce65f3e026e9bfe33e9c538fd3fbb2144fd9e", size = 24210, upload-time = "2024-07-26T18:15:03.762Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/b3/6b4067be973ae96ba0d615946e314c5ae35f9f993eca561b356540bb0c2b/alabaster-1.0.0-py3-none-any.whl", hash = "sha256:fc6786402dc3fcb2de3cabd5fe455a2db534b371124f1f21de8731783dec828b", size = 13929, upload-time = "2024-07-26T18:15:02.05Z" }, +] + [[package]] name = "async-generator" version = "1.10" @@ -24,6 +45,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/89/aa/ab0f7891a01eeb2d2e338ae8fecbe57fcebea1a24dbb64d45801bfab481d/attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308", size = 63397, upload-time = "2024-12-16T06:59:26.977Z" }, ] +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + [[package]] name = "base58" version = "2.1.1" @@ -33,6 +63,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4a/45/ec96b29162a402fc4c1c5512d114d7b3787b9d1c2ec241d9568b4816ee23/base58-2.1.1-py3-none-any.whl", hash = "sha256:11a36f4d3ce51dfc1043f3218591ac4eb1ceb172919cebe05b52a5bcc8d245c2", size = 5621, upload-time = "2021-10-30T22:12:16.658Z" }, ] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + [[package]] name = "bidict" version = "0.23.1" @@ -98,6 +141,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b0/5c/dbd00727a3dd165d7e0e8af40e630cd7e45d77b525a3218afaff8a87358e/blake3-1.0.8-cp314-cp314t-win_amd64.whl", hash = "sha256:421b99cdf1ff2d1bf703bc56c454f4b286fce68454dd8711abbcb5a0df90c19a", size = 215133, upload-time = "2025-10-14T06:47:16.069Z" }, ] +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422, upload-time = "2026-05-20T11:46:50.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134, upload-time = "2026-05-20T11:46:48.578Z" }, +] + [[package]] name = "cffi" version = "1.17.1" @@ -120,6 +172,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, ] +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271, upload-time = "2026-04-02T09:28:39.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627, upload-time = "2026-04-02T09:26:45.198Z" }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008, upload-time = "2026-04-02T09:26:46.824Z" }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303, upload-time = "2026-04-02T09:26:48.397Z" }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282, upload-time = "2026-04-02T09:26:49.684Z" }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595, upload-time = "2026-04-02T09:26:50.915Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986, upload-time = "2026-04-02T09:26:52.197Z" }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711, upload-time = "2026-04-02T09:26:53.49Z" }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036, upload-time = "2026-04-02T09:26:54.975Z" }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998, upload-time = "2026-04-02T09:26:56.303Z" }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056, upload-time = "2026-04-02T09:26:57.554Z" }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537, upload-time = "2026-04-02T09:26:58.843Z" }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176, upload-time = "2026-04-02T09:27:00.437Z" }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723, upload-time = "2026-04-02T09:27:02.021Z" }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085, upload-time = "2026-04-02T09:27:03.192Z" }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819, upload-time = "2026-04-02T09:27:04.454Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915, upload-time = "2026-04-02T09:27:05.971Z" }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234, upload-time = "2026-04-02T09:27:07.194Z" }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042, upload-time = "2026-04-02T09:27:08.749Z" }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706, upload-time = "2026-04-02T09:27:09.951Z" }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727, upload-time = "2026-04-02T09:27:11.175Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882, upload-time = "2026-04-02T09:27:12.446Z" }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860, upload-time = "2026-04-02T09:27:13.721Z" }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564, upload-time = "2026-04-02T09:27:15.272Z" }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276, upload-time = "2026-04-02T09:27:16.834Z" }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238, upload-time = "2026-04-02T09:27:18.229Z" }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189, upload-time = "2026-04-02T09:27:19.445Z" }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352, upload-time = "2026-04-02T09:27:20.79Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024, upload-time = "2026-04-02T09:27:22.063Z" }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869, upload-time = "2026-04-02T09:27:23.486Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541, upload-time = "2026-04-02T09:27:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634, upload-time = "2026-04-02T09:27:26.642Z" }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384, upload-time = "2026-04-02T09:27:28.271Z" }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133, upload-time = "2026-04-02T09:27:29.474Z" }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257, upload-time = "2026-04-02T09:27:30.793Z" }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851, upload-time = "2026-04-02T09:27:32.44Z" }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393, upload-time = "2026-04-02T09:27:34.03Z" }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251, upload-time = "2026-04-02T09:27:35.369Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609, upload-time = "2026-04-02T09:27:36.661Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014, upload-time = "2026-04-02T09:27:38.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979, upload-time = "2026-04-02T09:27:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238, upload-time = "2026-04-02T09:27:40.722Z" }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110, upload-time = "2026-04-02T09:27:42.33Z" }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824, upload-time = "2026-04-02T09:27:43.924Z" }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103, upload-time = "2026-04-02T09:27:45.348Z" }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194, upload-time = "2026-04-02T09:27:46.706Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827, upload-time = "2026-04-02T09:27:48.053Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168, upload-time = "2026-04-02T09:27:49.795Z" }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018, upload-time = "2026-04-02T09:27:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958, upload-time = "2026-04-02T09:28:37.794Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -150,6 +259,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/5a/18ad964b0086c6e62e2e7500f7edc89e3faa45033c71c1893d34eed2b2de/dnspython-2.8.0-py3-none-any.whl", hash = "sha256:01d9bbc4a2d76bf0db7c1f729812ded6d912bd318d3b1cf81d30c0f845dbf3af", size = 331094, upload-time = "2025-09-07T18:57:58.071Z" }, ] +[[package]] +name = "docutils" +version = "0.22.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/b6/03bb70946330e88ffec97aefd3ea75ba575cb2e762061e0e62a213befee8/docutils-0.22.4.tar.gz", hash = "sha256:4db53b1fde9abecbb74d91230d32ab626d94f6badfc575d6db9194a49df29968", size = 2291750, upload-time = "2025-12-18T19:00:26.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/10/5da547df7a391dcde17f59520a231527b8571e6f46fc8efb02ccb370ab12/docutils-0.22.4-py3-none-any.whl", hash = "sha256:d0013f540772d1420576855455d050a2180186c91c15779301ac2ccb3eeb68de", size = 633196, upload-time = "2025-12-18T19:00:18.077Z" }, +] + [[package]] name = "greenback" version = "1.2.1" @@ -197,6 +315,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, ] +[[package]] +name = "imagesize" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/e6/7bf14eeb8f8b7251141944835abd42eb20a658d89084b7e1f3e5fe394090/imagesize-2.0.0.tar.gz", hash = "sha256:8e8358c4a05c304f1fccf7ff96f036e7243a189e9e42e90851993c558cfe9ee3", size = 1773045, upload-time = "2026-03-03T14:18:29.941Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/53/fb7122b71361a0d121b669dcf3d31244ef75badbbb724af388948de543e2/imagesize-2.0.0-py2.py3-none-any.whl", hash = "sha256:5667c5bbb57ab3f1fa4bc366f4fbc971db3d5ed011fd2715fd8001f782718d96", size = 9441, upload-time = "2026-03-03T14:18:27.892Z" }, +] + [[package]] name = "importlib-metadata" version = "9.0.0" @@ -218,6 +345,70 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374", size = 5892, upload-time = "2023-01-07T11:08:09.864Z" }, ] +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + [[package]] name = "mmh3" version = "5.2.1" @@ -533,6 +724,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, ] +[[package]] +name = "pydata-sphinx-theme" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "accessible-pygments" }, + { name = "babel" }, + { name = "beautifulsoup4" }, + { name = "docutils" }, + { name = "pygments" }, + { name = "sphinx" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/81/b3fdc8b74d0cfed9e623a0fef9932376800da5daa1a85d1224cac4c131a3/pydata_sphinx_theme-0.18.0.tar.gz", hash = "sha256:b4abc95ab02600872e060db07c79e056e87b7ea653ab1ffd0e0b1fa75a3003d4", size = 5004260, upload-time = "2026-05-20T08:32:28.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/cd/e0eda602060f9dc99068f8e54490812d9d34ebb134043ff0ae594cf721a4/pydata_sphinx_theme-0.18.0-py3-none-any.whl", hash = "sha256:fbe5401f26642d487e3c5b6dfcbf69b3b1d579e80dcc479a429632abe0a13929", size = 6200747, upload-time = "2026-05-20T08:32:26.646Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -591,6 +800,30 @@ version = "1.2.2" source = { registry = "https://pypi.org/simple" } sdist = { url = "https://files.pythonhosted.org/packages/33/d0/9297d7d8dd74767b4d5560d834b30b2fff17d39987c23ed8656f476e0d9b/python-baseconv-1.2.2.tar.gz", hash = "sha256:0539f8bd0464013b05ad62e0a1673f0ac9086c76b43ebf9f833053527cd9931b", size = 4929, upload-time = "2019-04-04T19:28:57.17Z" } +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "roman-numerals" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/41dc953bbeb056c17d5f7a519f50fdf010bd0553be2d630bc69d1e022703/roman_numerals-4.1.0.tar.gz", hash = "sha256:1af8b147eb1405d5839e78aeb93131690495fe9da5c91856cb33ad55a7f1e5b2", size = 9077, upload-time = "2025-12-17T18:25:34.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/54/6f679c435d28e0a568d8e8a7c0a93a09010818634c3c3907fc98d8983770/roman_numerals-4.1.0-py3-none-any.whl", hash = "sha256:647ba99caddc2cc1e55a51e4360689115551bf4476d90e8162cf8c345fe233c7", size = 7676, upload-time = "2025-12-17T18:25:33.098Z" }, +] + [[package]] name = "ruff" version = "0.14.14" @@ -665,6 +898,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/08/b6/3a5a4f9952972791a9114ac01dfc123f0df79903577a3e0a7a404a695586/setproctitle-1.3.7-cp314-cp314t-win_amd64.whl", hash = "sha256:cbc388e3d86da1f766d8fc2e12682e446064c01cea9f88a88647cfe7c011de6a", size = 13469, upload-time = "2025-09-05T12:50:42.67Z" }, ] +[[package]] +name = "setuptools" +version = "82.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -683,6 +925,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "snowballstemmer" +version = "3.1.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/f8/0a71edf031f03c40db17503cb8ca78a69a171254e568e7db241b0ab57ea1/snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260", size = 123314, upload-time = "2026-06-03T00:56:40.194Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4c/07/2ebca9b11fb9be7340a818d8d6f63feaebb146be2c4afbd6061701d6df6e/snowballstemmer-3.1.1-py3-none-any.whl", hash = "sha256:7e207fa178741da09cdee59d3ecec3827ad5f92b1fc5c9ff3755b639f71f5752", size = 104164, upload-time = "2026-06-03T00:56:38.614Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -692,6 +943,148 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] +[[package]] +name = "soupsieve" +version = "2.8.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/2c/0a5f6f8ee0d5589e48c7640213ed5175d52cf540a06725b628cc1a45d6ce/soupsieve-2.8.4.tar.gz", hash = "sha256:e121fd02e975c695e4e9e8774a5ee35d74714b59307868dcc5319ad2d9e3328e", size = 121110, upload-time = "2026-05-24T13:55:57.154Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/f5/0c41cb68dcae6b7de4fac4188a3a9589e21fb31df21ea3a2e888db95e6c9/soupsieve-2.8.4-py3-none-any.whl", hash = "sha256:e7e6b0769c8f51ed59acab6e994b00621096cfb1c640a7509295987388fbaf65", size = 37304, upload-time = "2026-05-24T13:55:55.406Z" }, +] + +[[package]] +name = "sphinx" +version = "9.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "alabaster" }, + { name = "babel" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "docutils" }, + { name = "imagesize" }, + { name = "jinja2" }, + { name = "packaging" }, + { name = "pygments" }, + { name = "requests" }, + { name = "roman-numerals" }, + { name = "snowballstemmer" }, + { name = "sphinxcontrib-applehelp" }, + { name = "sphinxcontrib-devhelp" }, + { name = "sphinxcontrib-htmlhelp" }, + { name = "sphinxcontrib-jsmath" }, + { name = "sphinxcontrib-qthelp" }, + { name = "sphinxcontrib-serializinghtml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cd/bd/f08eb0f4eed5c83f1ba2a3bd18f7745a2b1525fad70660a1c00224ec468a/sphinx-9.1.0.tar.gz", hash = "sha256:7741722357dd75f8190766926071fed3bdc211c74dd2d7d4df5404da95930ddb", size = 8718324, upload-time = "2025-12-31T15:09:27.646Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/f7/b1884cb3188ab181fc81fa00c266699dab600f927a964df02ec3d5d1916a/sphinx-9.1.0-py3-none-any.whl", hash = "sha256:c84fdd4e782504495fe4f2c0b3413d6c2bf388589bb352d439b2a3bb99991978", size = 3921742, upload-time = "2025-12-31T15:09:25.561Z" }, +] + +[[package]] +name = "sphinx-copybutton" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/2b/a964715e7f5295f77509e59309959f4125122d648f86b4fe7d70ca1d882c/sphinx-copybutton-0.5.2.tar.gz", hash = "sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd", size = 23039, upload-time = "2023-04-14T08:10:22.998Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/48/1ea60e74949eecb12cdd6ac43987f9fd331156388dcc2319b45e2ebb81bf/sphinx_copybutton-0.5.2-py3-none-any.whl", hash = "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e", size = 13343, upload-time = "2023-04-14T08:10:20.844Z" }, +] + +[[package]] +name = "sphinx-design" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/13/7b/804f311da4663a4aecc6cf7abd83443f3d4ded970826d0c958edc77d4527/sphinx_design-0.7.0.tar.gz", hash = "sha256:d2a3f5b19c24b916adb52f97c5f00efab4009ca337812001109084a740ec9b7a", size = 2203582, upload-time = "2026-01-19T13:12:53.297Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/cf/45dd359f6ca0c3762ce0490f681da242f0530c49c81050c035c016bfdd3a/sphinx_design-0.7.0-py3-none-any.whl", hash = "sha256:f82bf179951d58f55dca78ab3706aeafa496b741a91b1911d371441127d64282", size = 2220350, upload-time = "2026-01-19T13:12:51.077Z" }, +] + +[[package]] +name = "sphinx-togglebutton" +version = "0.4.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docutils" }, + { name = "setuptools" }, + { name = "sphinx" }, + { name = "wheel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/be/169a0b0a8ad9588e8697c85e1d489aaaca7416073c2fc0267c360af5aae9/sphinx_togglebutton-0.4.5.tar.gz", hash = "sha256:c870dfbd3bc6e119b50ff9a37a64f8991902269e856728931c7d89877e8d4b3d", size = 18101, upload-time = "2026-03-27T13:50:41.984Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/2e/3dd55564928c5d61f92827d4b91307dde7911a40fbe0000645d73202eea9/sphinx_togglebutton-0.4.5-py3-none-any.whl", hash = "sha256:74eac6d2426110c3e1e6f989a98e07d7823141a335df1ad8a9d637bdf6a7af62", size = 44907, upload-time = "2026-03-27T13:50:40.94Z" }, +] + +[[package]] +name = "sphinxcontrib-applehelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/6e/b837e84a1a704953c62ef8776d45c3e8d759876b4a84fe14eba2859106fe/sphinxcontrib_applehelp-2.0.0.tar.gz", hash = "sha256:2f29ef331735ce958efa4734873f084941970894c6090408b079c61b2e1c06d1", size = 20053, upload-time = "2024-07-29T01:09:00.465Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/85/9ebeae2f76e9e77b952f4b274c27238156eae7979c5421fba91a28f4970d/sphinxcontrib_applehelp-2.0.0-py3-none-any.whl", hash = "sha256:4cd3f0ec4ac5dd9c17ec65e9ab272c9b867ea77425228e68ecf08d6b28ddbdb5", size = 119300, upload-time = "2024-07-29T01:08:58.99Z" }, +] + +[[package]] +name = "sphinxcontrib-devhelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/d2/5beee64d3e4e747f316bae86b55943f51e82bb86ecd325883ef65741e7da/sphinxcontrib_devhelp-2.0.0.tar.gz", hash = "sha256:411f5d96d445d1d73bb5d52133377b4248ec79db5c793ce7dbe59e074b4dd1ad", size = 12967, upload-time = "2024-07-29T01:09:23.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/7a/987e583882f985fe4d7323774889ec58049171828b58c2217e7f79cdf44e/sphinxcontrib_devhelp-2.0.0-py3-none-any.whl", hash = "sha256:aefb8b83854e4b0998877524d1029fd3e6879210422ee3780459e28a1f03a8a2", size = 82530, upload-time = "2024-07-29T01:09:21.945Z" }, +] + +[[package]] +name = "sphinxcontrib-htmlhelp" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/93/983afd9aa001e5201eab16b5a444ed5b9b0a7a010541e0ddfbbfd0b2470c/sphinxcontrib_htmlhelp-2.1.0.tar.gz", hash = "sha256:c9e2916ace8aad64cc13a0d233ee22317f2b9025b9cf3295249fa985cc7082e9", size = 22617, upload-time = "2024-07-29T01:09:37.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/7b/18a8c0bcec9182c05a0b3ec2a776bba4ead82750a55ff798e8d406dae604/sphinxcontrib_htmlhelp-2.1.0-py3-none-any.whl", hash = "sha256:166759820b47002d22914d64a075ce08f4c46818e17cfc9470a9786b759b19f8", size = 98705, upload-time = "2024-07-29T01:09:36.407Z" }, +] + +[[package]] +name = "sphinxcontrib-jsmath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/e8/9ed3830aeed71f17c026a07a5097edcf44b692850ef215b161b8ad875729/sphinxcontrib-jsmath-1.0.1.tar.gz", hash = "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8", size = 5787, upload-time = "2019-01-21T16:10:16.347Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, +] + +[[package]] +name = "sphinxcontrib-qthelp" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/68/bc/9104308fc285eb3e0b31b67688235db556cd5b0ef31d96f30e45f2e51cae/sphinxcontrib_qthelp-2.0.0.tar.gz", hash = "sha256:4fe7d0ac8fc171045be623aba3e2a8f613f8682731f9153bb2e40ece16b9bbab", size = 17165, upload-time = "2024-07-29T01:09:56.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/83/859ecdd180cacc13b1f7e857abf8582a64552ea7a061057a6c716e790fce/sphinxcontrib_qthelp-2.0.0-py3-none-any.whl", hash = "sha256:b18a828cdba941ccd6ee8445dbe72ffa3ef8cbe7505d8cd1fa0d42d3f2d5f3eb", size = 88743, upload-time = "2024-07-29T01:09:54.885Z" }, +] + +[[package]] +name = "sphinxcontrib-serializinghtml" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/44/6716b257b0aa6bfd51a1b31665d1c205fb12cb5ad56de752dfa15657de2f/sphinxcontrib_serializinghtml-2.0.0.tar.gz", hash = "sha256:e9d912827f872c029017a53f0ef2180b327c3f7fd23c87229f7a8e8b70031d4d", size = 16080, upload-time = "2024-07-29T01:10:09.332Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/a7/d2782e4e3f77c8450f727ba74a8f12756d5ba823d81b941f1b04da9d033a/sphinxcontrib_serializinghtml-2.0.0-py3-none-any.whl", hash = "sha256:6e2cb0eef194e10c27ec0023bfeb25badbbb5868244cf5bc5bdc04e4464bf331", size = 92072, upload-time = "2024-07-29T01:10:08.203Z" }, +] + +[[package]] +name = "sphinxext-opengraph" +version = "0.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/c0/eb6838e3bae624ce6c8b90b245d17e84252863150e95efdb88f92c8aa3fb/sphinxext_opengraph-0.13.0.tar.gz", hash = "sha256:103335d08567ad8468faf1425f575e3b698e9621f9323949a6c8b96d9793e80b", size = 1026875, upload-time = "2025-08-29T12:20:31.066Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/a4/66c1fd4f8fab88faf71cee04a945f9806ba0fef753f2cfc8be6353f64508/sphinxext_opengraph-0.13.0-py3-none-any.whl", hash = "sha256:936c07828edc9ad9a7b07908b29596dc84ed0b3ceaa77acdf51282d232d4d80e", size = 1004152, upload-time = "2025-08-29T12:20:29.072Z" }, +] + [[package]] name = "stackscope" version = "0.2.2" @@ -747,6 +1140,14 @@ devx = [ { name = "stackscope" }, { name = "typing-extensions" }, ] +docs = [ + { name = "pydata-sphinx-theme" }, + { name = "sphinx" }, + { name = "sphinx-copybutton" }, + { name = "sphinx-design" }, + { name = "sphinx-togglebutton" }, + { name = "sphinxext-opengraph" }, +] eventfd = [ { name = "cffi", marker = "python_full_version < '3.14'" }, ] @@ -803,6 +1204,14 @@ devx = [ { name = "stackscope", specifier = ">=0.2.2,<0.3" }, { name = "typing-extensions", specifier = ">=4.14.1" }, ] +docs = [ + { name = "pydata-sphinx-theme", specifier = ">=0.18,<0.19" }, + { name = "sphinx", specifier = ">=9.1,<10" }, + { name = "sphinx-copybutton", specifier = ">=0.5.2,<0.6" }, + { name = "sphinx-design", specifier = ">=0.7,<0.8" }, + { name = "sphinx-togglebutton", specifier = ">=0.4.5,<0.5" }, + { name = "sphinxext-opengraph", specifier = ">=0.13,<0.14" }, +] eventfd = [{ name = "cffi", marker = "python_full_version == '3.13.*'", specifier = ">=1.17.1" }] lint = [{ name = "ruff", specifier = ">=0.9.6" }] repl = [ @@ -875,6 +1284,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/00/d631e67a838026495268c2f6884f3711a15a9a2a96cd244fdaea53b823fb/typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76", size = 43906, upload-time = "2025-07-04T13:28:32.743Z" }, ] +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + [[package]] name = "varint" version = "1.0.2" @@ -890,6 +1308,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, ] +[[package]] +name = "wheel" +version = "0.47.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/62/75f18a0f03b4219c456652c7780e4d749b929eb605c098ce3a5b6b6bc081/wheel-0.47.0.tar.gz", hash = "sha256:cc72bd1009ba0cf63922e28f94d9d83b920aa2bb28f798a31d0691b02fa3c9b3", size = 63854, upload-time = "2026-04-22T15:51:27.727Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/1b/9e33c09813d65e248f7f773119148a612516a4bea93e9c6f545f78455b7c/wheel-0.47.0-py3-none-any.whl", hash = "sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced", size = 32218, upload-time = "2026-04-22T15:51:26.296Z" }, +] + [[package]] name = "wrapt" version = "1.17.2"