From 1e9b4fa7144c10f871a84797cc0e73f8c6319dd1 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Mon, 19 Oct 2026 17:09:30 +0200 Subject: [PATCH 1/2] feat(native): extract storage write path into planfile-native Rust package (ticket-226) Byte-identical libyaml-compatible YAML dump, JSON mirror and atomic sprint write in one native call; optional via PLANFILE_NATIVE=off|auto|verify with PyYAML fallback. 4.3-4.6x faster create/update on a 3000-ticket sprint. --- packages/planfile-native/Cargo.lock | 225 ++++ packages/planfile-native/Cargo.toml | 27 + packages/planfile-native/README.md | 67 ++ .../planfile-native/benches/bench_store.py | 94 ++ packages/planfile-native/pyproject.toml | 26 + .../python/planfile_native/__init__.py | 35 + packages/planfile-native/src/json.rs | 102 ++ packages/planfile-native/src/lib.rs | 262 +++++ packages/planfile-native/src/node.rs | 170 +++ packages/planfile-native/src/yaml.rs | 1038 +++++++++++++++++ packages/planfile-native/tests/test_parity.py | 210 ++++ planfile/core/fastio.py | 64 +- planfile/core/native.py | 104 ++ planfile/core/store.py | 8 +- project/TICKETS.md | 1 + project/ticket-226/README.md | 72 ++ project/ticket-226/intent.json | 28 + tests/test_native_integration.py | 129 ++ 18 files changed, 2652 insertions(+), 10 deletions(-) create mode 100644 packages/planfile-native/Cargo.lock create mode 100644 packages/planfile-native/Cargo.toml create mode 100644 packages/planfile-native/README.md create mode 100644 packages/planfile-native/benches/bench_store.py create mode 100644 packages/planfile-native/pyproject.toml create mode 100644 packages/planfile-native/python/planfile_native/__init__.py create mode 100644 packages/planfile-native/src/json.rs create mode 100644 packages/planfile-native/src/lib.rs create mode 100644 packages/planfile-native/src/node.rs create mode 100644 packages/planfile-native/src/yaml.rs create mode 100644 packages/planfile-native/tests/test_parity.py create mode 100644 planfile/core/native.py create mode 100644 project/ticket-226/README.md create mode 100644 project/ticket-226/intent.json create mode 100644 tests/test_native_integration.py diff --git a/packages/planfile-native/Cargo.lock b/packages/planfile-native/Cargo.lock new file mode 100644 index 0000000..9d4b8d4 --- /dev/null +++ b/packages/planfile-native/Cargo.lock @@ -0,0 +1,225 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "indoc" +version = "2.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37b2691796cffeb8a8cd305ac66e65841559f147f4e63231d0eafa4db5384d1" +dependencies = [ + "rustversion", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "planfile-native" +version = "0.1.0" +dependencies = [ + "pyo3", + "regex", +] + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pyo3" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f402062616ab18202ae8319da13fa4279883a2b8a9d9f83f20dbade813ce1884" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b14b5775b5ff446dd1056212d778012cbe8a0fbffd368029fd9e25b514479c38" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab5bcf04a2cdcbb50c7d6105de943f543f9ed92af55818fd17b660390fc8636" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fd24d897903a9e6d80b968368a34e1525aeb719d568dba8b3d4bfa5dc67d453" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" diff --git a/packages/planfile-native/Cargo.toml b/packages/planfile-native/Cargo.toml new file mode 100644 index 0000000..b07ecb6 --- /dev/null +++ b/packages/planfile-native/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "planfile-native" +version = "0.1.0" +edition = "2021" +authors = ["Tom Sapletta "] +description = "Native storage kernel for planfile: libyaml-compatible YAML dump, JSON mirror and atomic sprint writes" +license = "Apache-2.0" +publish = false + +[lib] +name = "_native" +crate-type = ["cdylib", "rlib"] + +[features] +# `extension-module` must be off for `cargo test` (links libpython); maturin turns it on. +extension-module = ["pyo3/extension-module"] + +[dependencies] +pyo3 = { version = "0.22", features = ["abi3-py310"] } +regex = { version = "1", default-features = false, features = ["std", "perf"] } + +[profile.release] +opt-level = 3 +lto = "fat" +codegen-units = 1 +strip = true +panic = "abort" diff --git a/packages/planfile-native/README.md b/packages/planfile-native/README.md new file mode 100644 index 0000000..d2f140a --- /dev/null +++ b/packages/planfile-native/README.md @@ -0,0 +1,67 @@ +# planfile-native + +Rust storage kernel for [planfile](../../README.md). It removes the write-path +bottleneck measured in `benches/bench_store.py`: every ticket mutation re-serialized the whole +sprint file with PyYAML and then re-walked it to build the JSON mirror. + +| Function | Replaces | Contract | +|---|---|---| +| `dump_yaml(data, allow_unicode)` | `yaml.dump(..., Dumper=CSafeDumper, default_flow_style=False)` | byte-identical output | +| `dump_json(data)` | `json.dumps(_json_safe(data), ensure_ascii=False)` | byte-identical output | +| `write_sprint(path, data, ...)` | `_write_yaml_atomic` (dump, tmp, fsync, rename, stat, mirror) | one call, files `0600` | +| `normalize(value)` | `fastio._json_safe` | same result and types | +| `atomic_write_text(path, text)` | `fastio._atomic_write_text` | same semantics | + +YAML stays the single source of truth. The emitter is a port of libyaml 0.2.5's block emitter +(`src/yaml.rs`, function names follow `emitter.c`) and the PyYAML implicit resolver +(`src/resolver.rs`), because sprint files live in users' repositories and any formatting change +would be a whole-file git diff. + +## Fail-closed fallback + +The extension accepts only what `yaml.SafeDumper` represents by *exact* type (`dict` with `str` +keys, `list`, `tuple`, `str`, `int`, `float`, `bool`, `None`, `date`, `datetime`). A subclass, +`set`, `bytes`, a container referenced twice (PyYAML would emit anchors), a cycle or a lone +surrogate raises `planfile_native.Unsupported` **before any file is touched**. planfile then runs +its pure-Python path, so behavior and errors are unchanged. + +## Runtime switches (no hardcoded policy) + +Read by `planfile/core/native.py`, not by the crate: + +| Variable | Values | Default | +|---|---|---| +| `PLANFILE_NATIVE` | `auto` use when importable, `off` never, `verify` use and compare with PyYAML, raise on mismatch | `auto` | +| `PLANFILE_NATIVE_FSYNC` | `1` / `0`, durability of the write | `1` | + +## Build and test + +```bash +cd packages/planfile-native +cargo test # unit tests +maturin develop --release # install into the active venv +pytest tests # differential tests against PyYAML +python benches/bench_store.py 3000 +``` + +`planfile` imports `planfile_native` optionally; it is not a hard dependency. + +## Standards alignment + +- `wellmanifest/performance`: evidence-bound change; baseline and candidate come from the same + workload (`benches/bench_store.py`), invariants are the parity tests, rollback is + `PLANFILE_NATIVE=off`. +- `wellmanifest/nohardcode`: policy (enable, fsync, mirror version and suffix) is injected from + Python or the environment, nothing is baked into the crate. +- `wellmanifest/data-plane` / `data-gateway`: see "Data-plane alignment" below. + +## Data-plane alignment + +planfile's storage follows the `sqlite-embedded-es-v1` shape of `wellmanifest/data-plane` only +in spirit: YAML sprint files are the authoritative writer per stream and the SQLite ticket index +(`planfile/core/sqlite_index.py`) is a rebuildable projection. This package is the single +*storage kernel* every write goes through (one operation layer, one authoritative writer, one +atomic commit point), which is the boundary a data-plane/data-gateway adapter would call. +It does **not** introduce an event store and does not change the source of truth; moving +planfile to event sourcing with YAML and SQLite as projections is a separate, larger decision. +The `data-gateway` one-line `typ: obiekt {zmienne}` format is not implemented here either. diff --git a/packages/planfile-native/benches/bench_store.py b/packages/planfile-native/benches/bench_store.py new file mode 100644 index 0000000..bf04fa0 --- /dev/null +++ b/packages/planfile-native/benches/bench_store.py @@ -0,0 +1,94 @@ +"""Representative planfile storage workload for baseline/candidate comparison. + + PLANFILE_NATIVE=off python benches/bench_store.py 3000 > baseline.json + PLANFILE_NATIVE=auto python benches/bench_store.py 3000 > candidate.json + +Both runs build the same store (N tickets in ``current``) in a fresh temporary directory and +time the same operations; every operation is sampled ``--samples`` times (default 7) and the +raw samples are printed as JSON so they can be attached as digest-bound evidence. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import statistics +import sys +import tempfile +import time +from pathlib import Path + +from planfile.core import native +from planfile.core.models import Ticket +from planfile.core.store import Store + + +def sample(fn, samples: int) -> list[float]: + fn() # warm-up, discarded + out = [] + for _ in range(samples): + start = time.perf_counter() + fn() + out.append((time.perf_counter() - start) * 1000.0) + return out + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("tickets", nargs="?", type=int, default=3000) + parser.add_argument("--samples", type=int, default=7) + args = parser.parse_args() + + root = Path(tempfile.mkdtemp(prefix="pfbench-")) + os.chdir(root) + store = Store(root) + store.init() + body = "d " * 200 + tickets = [ + Ticket(id=f"PLF-{i:05d}", name=f"Ticket {i} " + "lorem ipsum " * 8, description=body, priority="normal") + for i in range(args.tickets) + ] + start = time.perf_counter() + store.create_tickets_bulk(tickets) + bulk_ms = (time.perf_counter() - start) * 1000.0 + + counter = [args.tickets] + + def create_one() -> None: + counter[0] += 1 + store.create_ticket(Ticket(id=f"PLF-{counter[0]:05d}", name="new", priority="low")) + + def cold_list() -> None: + for mirror in (root / ".planfile").rglob("*.fast.json"): + mirror.unlink() + store.list_tickets("current") + + clock = [0] + + def update_one() -> None: + clock[0] += 1 + store.update_ticket("PLF-00042", name=f"renamed {clock[0]}") + + results = { + "update_ticket": sample(update_one, args.samples), + "create_ticket": sample(create_one, args.samples), + "list_tickets_warm": sample(lambda: store.list_tickets("current"), args.samples), + "get_ticket_warm": sample(lambda: store.get_ticket("PLF-00042"), args.samples), + "list_tickets_cold_no_mirror": sample(cold_list, max(3, args.samples // 2)), + } + report = { + "workload": {"tickets": args.tickets, "sprint_file_bytes": (root / ".planfile/sprints/current.yaml").stat().st_size}, + "native": {"mode": native.mode(), "available": native.available(), "enabled": native.enabled()}, + "platform": f"{platform.system()} {platform.machine()} python {platform.python_version()}", + "bulk_create_ms": bulk_ms, + "samples_ms": results, + "median_ms": {name: statistics.median(values) for name, values in results.items()}, + } + json.dump(report, sys.stdout, indent=2) + print() + + +if __name__ == "__main__": + main() diff --git a/packages/planfile-native/pyproject.toml b/packages/planfile-native/pyproject.toml new file mode 100644 index 0000000..b3740db --- /dev/null +++ b/packages/planfile-native/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["maturin>=1.5,<2.0"] +build-backend = "maturin" + +[project] +name = "planfile-native" +version = "0.1.0" +description = "Native storage kernel for planfile (YAML dump, JSON mirror, atomic sprint writes)" +authors = [{ name = "Tom Sapletta", email = "tom@sapletta.com" }] +license = { text = "Apache-2.0" } +readme = "README.md" +requires-python = ">=3.10" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: 3", +] + +[project.optional-dependencies] +test = ["pytest>=8.0", "pyyaml>=6.0"] + +[tool.maturin] +features = ["extension-module"] +module-name = "planfile_native._native" +python-source = "python" +binding = "pyo3" diff --git a/packages/planfile-native/python/planfile_native/__init__.py b/packages/planfile-native/python/planfile_native/__init__.py new file mode 100644 index 0000000..2bd83f0 --- /dev/null +++ b/packages/planfile-native/python/planfile_native/__init__.py @@ -0,0 +1,35 @@ +"""planfile-native: Rust storage kernel for planfile. + +The compiled extension is optional. ``AVAILABLE`` tells callers whether it could be imported; +when it is False they use their pure-Python implementation. +""" + +from __future__ import annotations + +try: + from . import _native + from ._native import ( # noqa: F401 + Unsupported, + atomic_write_text, + dump_json, + dump_yaml, + normalize, + write_sprint, + ) + + __version__ = _native.__version__ + AVAILABLE = True +except ImportError: # pragma: no cover - exercised only without the compiled wheel + AVAILABLE = False + __version__ = "0.0.0" + +__all__ = [ + "AVAILABLE", + "Unsupported", + "__version__", + "atomic_write_text", + "dump_json", + "dump_yaml", + "normalize", + "write_sprint", +] diff --git a/packages/planfile-native/src/json.rs b/packages/planfile-native/src/json.rs new file mode 100644 index 0000000..60ac38b --- /dev/null +++ b/packages/planfile-native/src/json.rs @@ -0,0 +1,102 @@ +//! `json.dumps(value, ensure_ascii=False)` for [`Node`] trees, byte-compatible with CPython's +//! encoder (default `", "` / `": "` separators, `NaN`/`Infinity` literals). +//! +//! The planfile JSON mirror only has to round-trip through `json.loads`, but keeping the exact +//! encoder format means a mirror written natively and one written by the Python fallback are +//! indistinguishable on disk. + +use crate::node::Node; +use std::fmt::Write; + +pub fn write_node(out: &mut String, node: &Node) { + match node { + Node::Null => out.push_str("null"), + Node::Bool(true) => out.push_str("true"), + Node::Bool(false) => out.push_str("false"), + Node::Int(text) => out.push_str(text), + Node::Float(repr) => out.push_str(match repr.as_str() { + "nan" => "NaN", + "inf" => "Infinity", + "-inf" => "-Infinity", + other => other, + }), + Node::Str(text) => write_string(out, text), + Node::Timestamp { json, .. } => write_string(out, json), + Node::Seq(items) => { + out.push('['); + for (index, item) in items.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_node(out, item); + } + out.push(']'); + } + Node::Map(entries) => { + out.push('{'); + for (index, (key, value)) in entries.iter().enumerate() { + if index > 0 { + out.push_str(", "); + } + write_string(out, key); + out.push_str(": "); + write_node(out, value); + } + out.push('}'); + } + } +} + +fn write_string(out: &mut String, text: &str) { + out.push('"'); + let mut run_start = 0; + for (index, c) in text.char_indices() { + let escape: Option<&str> = match c { + '"' => Some("\\\""), + '\\' => Some("\\\\"), + '\n' => Some("\\n"), + '\r' => Some("\\r"), + '\t' => Some("\\t"), + '\u{8}' => Some("\\b"), + '\u{c}' => Some("\\f"), + c if (c as u32) < 0x20 => None, + _ => continue, + }; + out.push_str(&text[run_start..index]); + match escape { + Some(sequence) => out.push_str(sequence), + None => { + let _ = write!(out, "\\u{:04x}", c as u32); + } + } + run_start = index + c.len_utf8(); + } + out.push_str(&text[run_start..]); + out.push('"'); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn matches_cpython_formatting() { + let node = Node::Map(vec![ + ( + "a".into(), + Node::Seq(vec![ + Node::Int("1".into()), + Node::Null, + Node::Float("nan".into()), + ]), + ), + ("b\"".into(), Node::Str("line\n\u{1}ż".into())), + ]); + let mut out = String::new(); + write_node(&mut out, &node); + assert_eq!( + out, + "{\"a\": [1, null, NaN], \"b\\\"\": \"line\\n\\u0001ż\"}" + ); + } +} diff --git a/packages/planfile-native/src/lib.rs b/packages/planfile-native/src/lib.rs new file mode 100644 index 0000000..f20d249 --- /dev/null +++ b/packages/planfile-native/src/lib.rs @@ -0,0 +1,262 @@ +//! planfile-native: the storage hot path of planfile in Rust. +//! +//! Exposed as `planfile_native._native`. Every function either produces exactly what the +//! pure-Python implementation produces or raises [`Unsupported`] before touching the filesystem, +//! so callers can always fall back to PyYAML without observable differences. + +mod json; +mod node; +mod yaml; + +use node::{Extractor, Node, NodeError}; +use pyo3::exceptions::{PyException, PyOSError}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyFloat, PyList, PyLong, PyString, PyTuple}; +use std::path::Path; + +pyo3::create_exception!( + _native, + Unsupported, + PyException, + "Raised when a value needs behavior only PyYAML implements; the caller must fall back." +); + +fn to_py_err(err: NodeError) -> PyErr { + match err { + NodeError::Unsupported(reason) => Unsupported::new_err(reason), + NodeError::Py(err) => err, + } +} + +fn extract_root(py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult { + Extractor::new(py)?.extract(data).map_err(to_py_err) +} + +fn render_yaml(node: &Node, allow_unicode: bool) -> PyResult { + // Sprint files are mostly string payload; one allocation of about the JSON size avoids + // repeated growth of the output buffer. + yaml::Emitter::dump(node, allow_unicode, 1 << 16) + .map_err(|yaml::EmitError::Unsupported(reason)| Unsupported::new_err(reason)) +} + +/// `yaml.dump(data, default_flow_style=False, allow_unicode=..., Dumper=CSafeDumper)`. +#[pyfunction] +#[pyo3(signature = (data, allow_unicode=false))] +fn dump_yaml(py: Python<'_>, data: &Bound<'_, PyAny>, allow_unicode: bool) -> PyResult { + let node = extract_root(py, data)?; + render_yaml(&node, allow_unicode) +} + +/// `json.dumps(_json_safe(data), ensure_ascii=False)`. +#[pyfunction] +fn dump_json(py: Python<'_>, data: &Bound<'_, PyAny>) -> PyResult { + let node = extract_root(py, data)?; + let mut out = String::new(); + json::write_node(&mut out, &node); + Ok(out) +} + +/// Writes `path` (YAML) and its JSON mirror in one call. +/// +/// Both documents are rendered before any file is touched. Returns +/// `(yaml_mtime_ns, mirror_error)`; a failing mirror write is reported, not raised, because the +/// mirror is only an optimization. +#[pyfunction] +#[pyo3(signature = (path, data, allow_unicode, mirror_version, mirror_suffix, fsync=true))] +fn write_sprint( + py: Python<'_>, + path: &str, + data: &Bound<'_, PyAny>, + allow_unicode: bool, + mirror_version: i64, + mirror_suffix: &str, + fsync: bool, +) -> PyResult<(i64, Option)> { + let node = extract_root(py, data)?; + let yaml_text = render_yaml(&node, allow_unicode)?; + let mut data_json = String::with_capacity(yaml_text.len()); + json::write_node(&mut data_json, &node); + drop(node); + + let target = Path::new(path).to_path_buf(); + let mirror_target = format!("{path}{mirror_suffix}"); + + py.allow_threads(move || { + fsio::atomic_write(&target, yaml_text.as_bytes(), fsync) + .map_err(|err| PyOSError::new_err(format!("{}: {err}", target.display())))?; + // Stat right after our own rename, mirroring the Python writer, so the mirror is stamped + // with the mtime of exactly the file this call produced. + let mtime = fsio::mtime_ns(&target) + .map_err(|err| PyOSError::new_err(format!("{}: {err}", target.display())))?; + let mirror = format!( + "{{\"version\": {mirror_version}, \"yaml_mtime_ns\": {mtime}, \"data\": {data_json}}}" + ); + let mirror_error = fsio::atomic_write(Path::new(&mirror_target), mirror.as_bytes(), fsync) + .err() + .map(|err| format!("{err}")); + Ok((mtime as i64, mirror_error)) + }) +} + +/// Atomically replaces `path` with `text` (UTF-8). Same contract as `_atomic_write_text`. +#[pyfunction] +#[pyo3(signature = (path, text, fsync=true))] +fn atomic_write_text(py: Python<'_>, path: &str, text: &str, fsync: bool) -> PyResult<()> { + let target = Path::new(path).to_path_buf(); + let bytes = text.as_bytes().to_vec(); + py.allow_threads(move || { + fsio::atomic_write(&target, &bytes, fsync) + .map_err(|err| PyOSError::new_err(format!("{}: {err}", target.display()))) + }) +} + +/// `fastio._json_safe`: dates become ISO strings, tuples become lists, containers are copied. +#[pyfunction] +fn normalize(py: Python<'_>, value: &Bound<'_, PyAny>) -> PyResult { + let module = py.import_bound("datetime")?; + let temporal = PyTuple::new_bound( + py, + [ + module.getattr("datetime")?, + module.getattr("date")?, + module.getattr("time")?, + ], + ); + normalize_inner(py, value, &temporal) +} + +fn normalize_inner( + py: Python<'_>, + value: &Bound<'_, PyAny>, + temporal: &Bound<'_, PyTuple>, +) -> PyResult { + if value.is_none() + || value.is_exact_instance_of::() + || value.is_exact_instance_of::() + || value.is_exact_instance_of::() + { + return Ok(value.clone().unbind()); + } + if let Ok(dict) = value.downcast::() { + let out = PyDict::new_bound(py); + for (key, item) in dict.iter() { + out.set_item( + normalize_inner(py, &key, temporal)?, + normalize_inner(py, &item, temporal)?, + )?; + } + return Ok(out.into_any().unbind()); + } + if value.downcast::().is_ok() || value.downcast::().is_ok() { + let out = PyList::empty_bound(py); + for item in value.iter()? { + out.append(normalize_inner(py, &item?, temporal)?)?; + } + return Ok(out.into_any().unbind()); + } + if value.is_instance(temporal)? { + return Ok(value.call_method0("isoformat")?.unbind()); + } + Ok(value.clone().unbind()) +} + +#[pymodule] +fn _native(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add("Unsupported", m.py().get_type_bound::())?; + m.add("__version__", env!("CARGO_PKG_VERSION"))?; + m.add_function(wrap_pyfunction!(dump_yaml, m)?)?; + m.add_function(wrap_pyfunction!(dump_json, m)?)?; + m.add_function(wrap_pyfunction!(write_sprint, m)?)?; + m.add_function(wrap_pyfunction!(atomic_write_text, m)?)?; + m.add_function(wrap_pyfunction!(normalize, m)?)?; + Ok(()) +} + +/// Crash-safe file replacement: temp file in the target directory, optional fsync, rename. +mod fsio { + use std::fs::{self, OpenOptions}; + use std::io::Write; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{SystemTime, UNIX_EPOCH}; + + static COUNTER: AtomicU64 = AtomicU64::new(0); + + fn temp_path(target: &Path) -> PathBuf { + let name = target + .file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_default(); + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0); + let unique = COUNTER.fetch_add(1, Ordering::Relaxed); + let dir = target + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + dir.join(format!( + ".{name}.{:x}{:x}{:x}.tmp", + std::process::id(), + nanos, + unique + )) + } + + /// Writes `content` to `target` atomically. The new file is `0600` like `tempfile.mkstemp`, + /// which is what the previous Python implementation produced. + pub fn atomic_write(target: &Path, content: &[u8], fsync: bool) -> std::io::Result<()> { + if let Some(parent) = target.parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent)?; + } + } + let tmp = temp_path(target); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let result = (|| { + let mut file = options.open(&tmp)?; + file.write_all(content)?; + file.flush()?; + if fsync { + file.sync_all()?; + } + drop(file); + fs::rename(&tmp, target) + })(); + if result.is_err() { + let _ = fs::remove_file(&tmp); + } + result + } + + /// `st_mtime_ns` of `path` as an unsigned nanosecond count (matches `os.stat`). + pub fn mtime_ns(path: &Path) -> std::io::Result { + use std::os::unix::fs::MetadataExt; + let meta = fs::metadata(path)?; + Ok(meta.mtime() as i128 * 1_000_000_000 + meta.mtime_nsec() as i128) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn replaces_file_and_leaves_no_temp() { + let dir = std::env::temp_dir().join(format!("pfn-atomic-{}", std::process::id())); + let target = dir.join("nested").join("a.yaml"); + atomic_write(&target, b"one", true).unwrap(); + atomic_write(&target, b"two", false).unwrap(); + assert_eq!(fs::read(&target).unwrap(), b"two"); + let leftovers = fs::read_dir(target.parent().unwrap()).unwrap().count(); + assert_eq!(leftovers, 1); + fs::remove_dir_all(dir).unwrap(); + } + } +} diff --git a/packages/planfile-native/src/node.rs b/packages/planfile-native/src/node.rs new file mode 100644 index 0000000..3a459a0 --- /dev/null +++ b/packages/planfile-native/src/node.rs @@ -0,0 +1,170 @@ +//! Plain-Rust value tree extracted from Python objects. +//! +//! Only types that `yaml.SafeDumper` (and `json.dumps`) represent without special handling are +//! accepted, and only by *exact* type, mirroring `Representer.yaml_representers` lookup. Anything +//! else, an object referenced twice (PyYAML would emit anchors) or a cycle yields +//! [`NodeError::Unsupported`], and the Python caller falls back to PyYAML so behavior (including +//! errors) stays identical. + +use pyo3::prelude::*; +use pyo3::sync::GILOnceCell; +use pyo3::types::{PyBool, PyDict, PyFloat, PyList, PyLong, PyString, PyTuple}; +use std::collections::HashSet; + +/// Containers nested deeper than this are left to PyYAML instead of risking native stack use. +const MAX_DEPTH: usize = 256; + +#[derive(Debug)] +pub enum Node { + Null, + Bool(bool), + Int(String), + /// `repr(float)` exactly as Python prints it. + Float(String), + Str(String), + Timestamp { + yaml: String, + json: String, + }, + Seq(Vec), + /// Insertion-ordered; the YAML emitter sorts, the JSON writer does not. + Map(Vec<(String, Node)>), +} + +#[derive(Debug)] +pub enum NodeError { + Unsupported(String), + Py(PyErr), +} + +impl From for NodeError { + fn from(err: PyErr) -> Self { + NodeError::Py(err) + } +} + +impl From> for NodeError { + fn from(err: pyo3::DowncastError<'_, '_>) -> Self { + NodeError::Py(err.into()) + } +} + +fn unsupported(reason: impl Into) -> Result { + Err(NodeError::Unsupported(reason.into())) +} + +struct DateTypes { + datetime: Py, + date: Py, +} + +fn date_types(py: Python<'_>) -> PyResult<&DateTypes> { + static CELL: GILOnceCell = GILOnceCell::new(); + CELL.get_or_try_init(py, || { + let module = py.import_bound("datetime")?; + Ok(DateTypes { + datetime: module.getattr("datetime")?.unbind(), + date: module.getattr("date")?.unbind(), + }) + }) +} + +pub struct Extractor<'py> { + py: Python<'py>, + seen: HashSet, + dates: &'py DateTypes, +} + +impl<'py> Extractor<'py> { + pub fn new(py: Python<'py>) -> PyResult { + Ok(Self { + py, + seen: HashSet::new(), + dates: date_types(py)?, + }) + } + + pub fn extract(&mut self, obj: &Bound<'py, PyAny>) -> Result { + self.visit(obj, 0) + } + + fn visit(&mut self, obj: &Bound<'py, PyAny>, depth: usize) -> Result { + if obj.is_none() { + return Ok(Node::Null); + } + if obj.is_exact_instance_of::() { + return Ok(Node::Bool(obj.is_truthy()?)); + } + if obj.is_exact_instance_of::() { + let text = obj.downcast::()?.to_str().map_err(|_| { + NodeError::Unsupported("string is not valid UTF-8 (lone surrogate)".into()) + })?; + return Ok(Node::Str(text.to_owned())); + } + if obj.is_exact_instance_of::() { + return Ok(Node::Int(obj.str()?.to_string_lossy().into_owned())); + } + if obj.is_exact_instance_of::() { + return Ok(Node::Float(obj.repr()?.to_string_lossy().into_owned())); + } + if obj.is_exact_instance_of::() { + return self.visit_dict(obj, depth); + } + if obj.is_exact_instance_of::() || obj.is_exact_instance_of::() { + return self.visit_seq(obj, depth); + } + let ty = obj.get_type(); + if ty.is(self.dates.datetime.bind(self.py)) { + let yaml = obj.call_method1("isoformat", (" ",))?.extract::()?; + let json = obj.call_method0("isoformat")?.extract::()?; + return Ok(Node::Timestamp { yaml, json }); + } + if ty.is(self.dates.date.bind(self.py)) { + let text = obj.call_method0("isoformat")?.extract::()?; + return Ok(Node::Timestamp { + yaml: text.clone(), + json: text, + }); + } + unsupported(format!("unsupported type {}", ty.name()?)) + } + + /// Registers a container. PyYAML anchors any container reached twice, so the identity stays + /// in `seen` for the whole walk. + fn enter(&mut self, obj: &Bound<'py, PyAny>, depth: usize) -> Result<(), NodeError> { + if depth >= MAX_DEPTH { + return unsupported("maximum nesting depth exceeded"); + } + let id = obj.as_ptr() as usize; + if !self.seen.insert(id) { + return unsupported("container referenced more than once (YAML anchors required)"); + } + Ok(()) + } + + fn visit_dict(&mut self, obj: &Bound<'py, PyAny>, depth: usize) -> Result { + self.enter(obj, depth)?; + let dict = obj.downcast::()?; + let mut entries = Vec::with_capacity(dict.len()); + for (key, value) in dict.iter() { + if !key.is_exact_instance_of::() { + return unsupported("non-string mapping key"); + } + let key = key + .downcast::()? + .to_str() + .map_err(|_| NodeError::Unsupported("mapping key is not valid UTF-8".into()))?; + entries.push((key.to_owned(), self.visit(&value, depth + 1)?)); + } + Ok(Node::Map(entries)) + } + + fn visit_seq(&mut self, obj: &Bound<'py, PyAny>, depth: usize) -> Result { + self.enter(obj, depth)?; + let mut items = Vec::new(); + for item in obj.iter()? { + items.push(self.visit(&item?, depth + 1)?); + } + Ok(Node::Seq(items)) + } +} diff --git a/packages/planfile-native/src/yaml.rs b/packages/planfile-native/src/yaml.rs new file mode 100644 index 0000000..bcc4e4a --- /dev/null +++ b/packages/planfile-native/src/yaml.rs @@ -0,0 +1,1038 @@ +//! Port of the libyaml 0.2.5 block emitter, restricted to what `yaml.CSafeDumper` produces for +//! `default_flow_style=False, sort_keys=True` over JSON-like data. +//! +//! The goal is byte-identical output: sprint files are tracked in users' repositories, so a +//! different serializer would show up as whole-file git diffs. The function names follow libyaml +//! (`write_indent`, `write_indicator`, `analyze_scalar`, ...) so the port can be audited against +//! `emitter.c`. Differential tests against PyYAML live in `tests/test_parity.py`. + +use crate::node::Node; +use regex::Regex; +use std::sync::OnceLock; + +// ---- PyYAML implicit resolver (YAML 1.1) ---- +// PyYAML 6 implicit scalar resolver (YAML 1.1 rules). +// +// `yaml.resolver.Resolver` decides, per plain scalar, whether the text would be read back as a +// non-string (bool, int, float, null, timestamp, merge, value). The emitter quotes a string when +// that happens, so these patterns must stay identical to PyYAML's. Dispatch is on the first +// character exactly as `BaseResolver.resolve` does. + +struct Patterns { + boolean: Regex, + float: Regex, + int: Regex, + null: Regex, + timestamp: Regex, +} + +fn patterns() -> &'static Patterns { + static CELL: OnceLock = OnceLock::new(); + CELL.get_or_init(|| Patterns { + boolean: Regex::new( + r"^(?:yes|Yes|YES|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF)$", + ) + .expect("bool pattern"), + float: Regex::new(concat!( + r"^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)?", + r"|\.[0-9][0-9_]*(?:[eE][-+][0-9]+)?", + r"|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*", + r"|[-+]?\.(?:inf|Inf|INF)", + r"|\.(?:nan|NaN|NAN))$" + )) + .expect("float pattern"), + int: Regex::new(concat!( + r"^(?:[-+]?0b[0-1_]+", + r"|[-+]?0[0-7_]+", + r"|[-+]?(?:0|[1-9][0-9_]*)", + r"|[-+]?0x[0-9a-fA-F_]+", + r"|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$" + )) + .expect("int pattern"), + null: Regex::new(r"^(?:~|null|Null|NULL|)$").expect("null pattern"), + timestamp: Regex::new(concat!( + r"^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]", + r"|[0-9][0-9][0-9][0-9]-[0-9][0-9]?-[0-9][0-9]?", + r"(?:[Tt]|[ \t]+)[0-9][0-9]?", + r":[0-9][0-9]:[0-9][0-9](?:\.[0-9]*)?", + r"(?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$" + )) + .expect("timestamp pattern"), + }) +} + +/// True when PyYAML's implicit resolver would assign a non-`str` tag to `value`. +/// +/// Python's `$` also matches before one trailing newline, the `regex` crate's does not, so a +/// single trailing `\n` is stripped before matching. +fn resolves_to_non_str(value: &str) -> bool { + let first = match value.chars().next() { + None => return true, // '' resolves to null + Some(c) => c, + }; + let text = value.strip_suffix('\n').unwrap_or(value); + let p = patterns(); + match first { + '-' | '+' => p.float.is_match(text) || p.int.is_match(text), + '.' => p.float.is_match(text), + '0'..='9' => p.float.is_match(text) || p.int.is_match(text) || p.timestamp.is_match(text), + 'y' | 'Y' | 'f' | 'F' | 'o' | 'O' | 't' | 'T' => p.boolean.is_match(text), + 'n' | 'N' => p.boolean.is_match(text) || p.null.is_match(text), + '~' => p.null.is_match(text), + '<' => text == "<<", + '=' => text == "=", + '!' | '&' | '*' => text.chars().count() == 1, + _ => false, + } +} + +/// libyaml `best_width` after `yaml_emitter_emit_stream_start` normalisation (`width=None`). +const BEST_WIDTH: usize = 80; +const BEST_INDENT: i32 = 2; +/// `yaml_emitter_check_simple_key` refuses keys longer than this many bytes. +const MAX_SIMPLE_KEY_BYTES: usize = 128; + +#[derive(Debug)] +pub enum EmitError { + /// The value needs tags or anchors the port does not reproduce; PyYAML must handle it. + Unsupported(String), +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Style { + Plain, + Single, + Double, +} + +struct Analysis { + multiline: bool, + flow_plain_allowed: bool, + block_plain_allowed: bool, + single_quoted_allowed: bool, +} + +fn is_break(c: char) -> bool { + matches!(c, '\r' | '\n' | '\u{85}' | '\u{2028}' | '\u{2029}') +} + +fn is_blankz(c: Option) -> bool { + match c { + None => true, + Some(c) => c == ' ' || c == '\t' || c == '\0' || is_break(c), + } +} + +/// libyaml `IS_PRINTABLE` expressed on code points (it is defined on UTF-8 lead bytes, which +/// makes every supplementary-plane character "unprintable"). +fn is_printable(c: char) -> bool { + let v = c as u32; + v == 0x0A + || (0x20..=0x7E).contains(&v) + || (0xA0..=0xD7FF).contains(&v) + || ((0xE000..=0xFFFD).contains(&v) && v != 0xFEFF) +} + +fn analyze_scalar(value: &str, unicode: bool) -> Analysis { + if value.is_ascii() { + return analyze_ascii(value.as_bytes()); + } + analyze_general(value, unicode) +} + +/// Byte-wise twin of [`analyze_general`] for ASCII text (the overwhelmingly common case). Kept +/// in lock-step by `ascii_analysis_equals_general_analysis`. +fn analyze_ascii(bytes: &[u8]) -> Analysis { + let n = bytes.len(); + if n == 0 { + return Analysis { + multiline: false, + flow_plain_allowed: false, + block_plain_allowed: true, + single_quoted_allowed: true, + }; + } + let blankz = |b: u8| matches!(b, b' ' | b'\t' | b'\n' | b'\r' | 0); + let mut block_indicators = false; + let mut flow_indicators = false; + let mut line_breaks = false; + let mut special_characters = false; + let (mut leading_space, mut leading_break, mut trailing_space, mut trailing_break) = + (false, false, false, false); + let (mut break_space, mut space_break) = (false, false); + let (mut previous_space, mut previous_break) = (false, false); + if bytes.starts_with(b"---") || bytes.starts_with(b"...") { + block_indicators = true; + flow_indicators = true; + } + let mut preceded_by_whitespace = true; + for (i, &c) in bytes.iter().enumerate() { + let first = i == 0; + let last = i + 1 == n; + let followed_by_whitespace = last || blankz(bytes[i + 1]); + if first { + if matches!( + c, + b'#' | b',' + | b'[' + | b']' + | b'{' + | b'}' + | b'&' + | b'*' + | b'!' + | b'|' + | b'>' + | b'\'' + | b'"' + | b'%' + | b'@' + | b'`' + ) { + flow_indicators = true; + block_indicators = true; + } + if c == b'?' || c == b':' { + flow_indicators = true; + if followed_by_whitespace { + block_indicators = true; + } + } + if c == b'-' && followed_by_whitespace { + flow_indicators = true; + block_indicators = true; + } + } else { + if matches!(c, b',' | b'?' | b'[' | b']' | b'{' | b'}') { + flow_indicators = true; + } + if c == b':' { + flow_indicators = true; + if followed_by_whitespace { + block_indicators = true; + } + } + if c == b'#' && preceded_by_whitespace { + flow_indicators = true; + block_indicators = true; + } + } + if !(c == b'\n' || (0x20..=0x7E).contains(&c)) { + special_characters = true; + } + let is_brk = c == b'\n' || c == b'\r'; + if is_brk { + line_breaks = true; + } + if c == b' ' { + leading_space |= first; + trailing_space |= last; + break_space |= previous_break; + previous_space = true; + previous_break = false; + } else if is_brk { + leading_break |= first; + trailing_break |= last; + space_break |= previous_space; + previous_space = false; + previous_break = true; + } else { + previous_space = false; + previous_break = false; + } + preceded_by_whitespace = blankz(c); + } + finish_analysis( + line_breaks, + flow_indicators, + block_indicators, + special_characters, + leading_space || leading_break || trailing_space || trailing_break, + break_space, + space_break, + ) +} + +fn analyze_general(value: &str, unicode: bool) -> Analysis { + if value.is_empty() { + return Analysis { + multiline: false, + flow_plain_allowed: false, + block_plain_allowed: true, + single_quoted_allowed: true, + }; + } + + let mut block_indicators = false; + let mut flow_indicators = false; + let mut line_breaks = false; + let mut special_characters = false; + let mut leading_space = false; + let mut leading_break = false; + let mut trailing_space = false; + let mut trailing_break = false; + let mut break_space = false; + let mut space_break = false; + let mut previous_space = false; + let mut previous_break = false; + + if value.starts_with("---") || value.starts_with("...") { + block_indicators = true; + flow_indicators = true; + } + + let mut preceded_by_whitespace = true; + let mut iter = value.char_indices().peekable(); + let mut followed_by_whitespace = { + let mut look = value.chars(); + look.next(); + is_blankz(look.next()) + }; + let mut first = true; + + while let Some((index, c)) = iter.next() { + let last = iter.peek().is_none(); + if first { + if matches!( + c, + '#' | ',' + | '[' + | ']' + | '{' + | '}' + | '&' + | '*' + | '!' + | '|' + | '>' + | '\'' + | '"' + | '%' + | '@' + | '`' + ) { + flow_indicators = true; + block_indicators = true; + } + if c == '?' || c == ':' { + flow_indicators = true; + if followed_by_whitespace { + block_indicators = true; + } + } + if c == '-' && followed_by_whitespace { + flow_indicators = true; + block_indicators = true; + } + } else { + if matches!(c, ',' | '?' | '[' | ']' | '{' | '}') { + flow_indicators = true; + } + if c == ':' { + flow_indicators = true; + if followed_by_whitespace { + block_indicators = true; + } + } + if c == '#' && preceded_by_whitespace { + flow_indicators = true; + block_indicators = true; + } + } + + if !is_printable(c) || (!c.is_ascii() && !unicode) { + special_characters = true; + } + if is_break(c) { + line_breaks = true; + } + + if c == ' ' { + if first { + leading_space = true; + } + if last { + trailing_space = true; + } + if previous_break { + break_space = true; + } + previous_space = true; + previous_break = false; + } else if is_break(c) { + if first { + leading_break = true; + } + if last { + trailing_break = true; + } + if previous_space { + space_break = true; + } + previous_space = false; + previous_break = true; + } else { + previous_space = false; + previous_break = false; + } + + preceded_by_whitespace = is_blankz(Some(c)); + if !last { + let rest = &value[index + c.len_utf8()..]; + let mut look = rest.chars(); + look.next(); + followed_by_whitespace = is_blankz(look.next()); + } + first = false; + } + + finish_analysis( + line_breaks, + flow_indicators, + block_indicators, + special_characters, + leading_space || leading_break || trailing_space || trailing_break, + break_space, + space_break, + ) +} + +fn finish_analysis( + line_breaks: bool, + flow_indicators: bool, + block_indicators: bool, + special_characters: bool, + edge_whitespace: bool, + break_space: bool, + space_break: bool, +) -> Analysis { + let mut analysis = Analysis { + multiline: line_breaks, + flow_plain_allowed: true, + block_plain_allowed: true, + single_quoted_allowed: true, + }; + if edge_whitespace { + analysis.flow_plain_allowed = false; + analysis.block_plain_allowed = false; + } + if break_space { + analysis.flow_plain_allowed = false; + analysis.block_plain_allowed = false; + analysis.single_quoted_allowed = false; + } + if space_break || special_characters { + analysis.flow_plain_allowed = false; + analysis.block_plain_allowed = false; + analysis.single_quoted_allowed = false; + } + if line_breaks { + analysis.flow_plain_allowed = false; + analysis.block_plain_allowed = false; + } + if flow_indicators { + analysis.flow_plain_allowed = false; + } + if block_indicators { + analysis.block_plain_allowed = false; + } + analysis +} + +pub struct Emitter { + out: String, + column: usize, + indent: i32, + indents: Vec, + whitespace: bool, + indention: bool, + flow_level: usize, + mapping_context: bool, + simple_key_context: bool, + unicode: bool, +} + +/// How a scalar's text relates to its tag, i.e. `plain_implicit` / `quoted_implicit` of the +/// libyaml scalar event. +struct ScalarEvent<'a> { + text: &'a str, + plain_implicit: bool, + /// Non-strings carry a non-`str` tag; only plain style can omit it. + is_str: bool, +} + +impl Emitter { + pub fn new(unicode: bool, capacity: usize) -> Self { + Emitter { + out: String::with_capacity(capacity), + column: 0, + indent: -1, + indents: Vec::new(), + whitespace: true, + indention: true, + flow_level: 0, + mapping_context: false, + simple_key_context: false, + unicode, + } + } + + /// Serializes one document; the root must be a mapping or a sequence. + pub fn dump(root: &Node, unicode: bool, capacity_hint: usize) -> Result { + if !matches!(root, Node::Map(_) | Node::Seq(_)) { + return Err(EmitError::Unsupported( + "root must be a mapping or sequence".into(), + )); + } + let mut emitter = Emitter::new(unicode, capacity_hint); + emitter.emit_node(root, false, false)?; + // yaml_emitter_emit_document_end: write_indent terminates the last line. + emitter.write_indent(); + Ok(emitter.out) + } + + // ---- low level writers (emitter.c: PUT, PUT_BREAK, write_indent, write_indicator) ---- + + fn put(&mut self, c: char) { + self.out.push(c); + self.column += 1; + } + + fn put_break(&mut self) { + self.out.push('\n'); + self.column = 0; + } + + fn write_indent(&mut self) { + let indent = if self.indent >= 0 { + self.indent as usize + } else { + 0 + }; + if !self.indention || self.column > indent || (self.column == indent && !self.whitespace) { + self.put_break(); + } + while self.column < indent { + self.put(' '); + } + self.whitespace = true; + self.indention = true; + } + + fn write_indicator( + &mut self, + indicator: &str, + need_whitespace: bool, + is_whitespace: bool, + is_indention: bool, + ) { + if need_whitespace && !self.whitespace { + self.put(' '); + } + self.out.push_str(indicator); + self.column += indicator.chars().count(); + self.whitespace = is_whitespace; + self.indention = self.indention && is_indention; + } + + fn increase_indent(&mut self, flow: bool, indentless: bool) { + self.indents.push(self.indent); + if self.indent < 0 { + self.indent = if flow { BEST_INDENT } else { 0 }; + } else if !indentless { + self.indent += BEST_INDENT; + } + } + + fn pop_indent(&mut self) { + self.indent = self.indents.pop().unwrap_or(-1); + } + + // ---- node dispatch (emit_node and the block collection states) ---- + + fn emit_node(&mut self, node: &Node, mapping: bool, simple_key: bool) -> Result<(), EmitError> { + self.mapping_context = mapping; + self.simple_key_context = simple_key; + match node { + Node::Seq(items) => { + self.emit_sequence(items)?; + Ok(()) + } + Node::Map(entries) => self.emit_mapping(entries), + Node::Null => self.emit_non_str("null"), + Node::Bool(true) => self.emit_non_str("true"), + Node::Bool(false) => self.emit_non_str("false"), + Node::Int(text) => self.emit_non_str(text), + Node::Float(repr) => { + let text = float_text(repr); + self.emit_non_str(&text) + } + Node::Timestamp { yaml, .. } => self.emit_non_str(yaml), + Node::Str(text) => self.emit_scalar(&ScalarEvent { + text, + plain_implicit: !resolves_to_non_str(text), + is_str: true, + }), + } + } + + fn emit_non_str(&mut self, text: &str) -> Result<(), EmitError> { + let plain_implicit = true; // the representer's own text always resolves to its tag + self.emit_scalar(&ScalarEvent { + text, + plain_implicit, + is_str: false, + }) + } + + fn emit_sequence(&mut self, items: &[Node]) -> Result<(), EmitError> { + if self.flow_level > 0 || items.is_empty() { + self.write_indicator("[", true, true, false); + self.increase_indent(true, false); + self.flow_level += 1; + self.flow_level -= 1; + self.pop_indent(); + self.write_indicator("]", false, false, false); + return Ok(()); + } + let indentless = self.mapping_context && !self.indention; + self.increase_indent(false, indentless); + for item in items { + self.write_indent(); + self.write_indicator("-", true, false, true); + self.emit_node(item, false, false)?; + } + self.pop_indent(); + Ok(()) + } + + fn emit_mapping(&mut self, entries: &[(String, Node)]) -> Result<(), EmitError> { + if self.flow_level > 0 || entries.is_empty() { + self.write_indicator("{", true, true, false); + self.increase_indent(true, false); + self.flow_level += 1; + self.flow_level -= 1; + self.pop_indent(); + self.write_indicator("}", false, false, false); + return Ok(()); + } + // PyYAML sorts items before representing them (sort_keys=True). + let mut order: Vec<&(String, Node)> = entries.iter().collect(); + order.sort_by(|a, b| a.0.cmp(&b.0)); + + self.increase_indent(false, false); + for (key, value) in order.iter().map(|e| (&e.0, &e.1)) { + self.write_indent(); + let key_analysis = analyze_scalar(key, self.unicode); + let simple = !key_analysis.multiline && key.len() <= MAX_SIMPLE_KEY_BYTES; + let key_event = ScalarEvent { + text: key, + plain_implicit: !resolves_to_non_str(key), + is_str: true, + }; + if simple { + self.mapping_context = true; + self.simple_key_context = true; + self.emit_analyzed_scalar(&key_event, &key_analysis)?; + self.write_indicator(":", false, false, false); + } else { + self.write_indicator("?", true, false, true); + self.mapping_context = true; + self.simple_key_context = false; + self.emit_analyzed_scalar(&key_event, &key_analysis)?; + self.write_indent(); + self.write_indicator(":", true, false, true); + } + self.emit_node(value, true, false)?; + } + self.pop_indent(); + Ok(()) + } + + // ---- scalars (select_scalar_style, process_scalar, write_*_scalar) ---- + + fn emit_scalar(&mut self, event: &ScalarEvent<'_>) -> Result<(), EmitError> { + let analysis = analyze_scalar(event.text, self.unicode); + self.emit_analyzed_scalar(event, &analysis) + } + + fn emit_analyzed_scalar( + &mut self, + event: &ScalarEvent<'_>, + analysis: &Analysis, + ) -> Result<(), EmitError> { + let style = self.select_style(event, analysis); + if !event.is_str && style != Style::Plain { + return Err(EmitError::Unsupported( + "non-string scalar needs an explicit tag".into(), + )); + } + self.increase_indent(false, false); + let allow_breaks = !self.simple_key_context; + match style { + Style::Plain => self.write_plain(event.text, allow_breaks), + Style::Single => self.write_single_quoted(event.text, allow_breaks), + Style::Double => self.write_double_quoted(event.text, allow_breaks), + } + self.pop_indent(); + Ok(()) + } + + fn select_style(&self, event: &ScalarEvent<'_>, analysis: &Analysis) -> Style { + let mut style = Style::Plain; + if self.simple_key_context && analysis.multiline { + style = Style::Double; + } + if style == Style::Plain { + let plain_allowed = if self.flow_level > 0 { + analysis.flow_plain_allowed + } else { + analysis.block_plain_allowed + }; + if !plain_allowed { + style = Style::Single; + } + if event.text.is_empty() && (self.flow_level > 0 || self.simple_key_context) { + style = Style::Single; + } + if !event.plain_implicit { + style = Style::Single; + } + } + if style == Style::Single && !analysis.single_quoted_allowed { + style = Style::Double; + } + style + } + + /// Appends a run of characters that need no per-character decision. + fn put_run(&mut self, run: &str) { + if run.is_empty() { + return; + } + // The per-character writers clear `indention` after every ordinary character. + self.indention = false; + self.out.push_str(run); + self.column += if run.is_ascii() { + run.len() + } else { + run.chars().count() + }; + } + + fn write_plain(&mut self, value: &str, allow_breaks: bool) { + if !self.whitespace && (!value.is_empty() || self.flow_level > 0) { + self.put(' '); + } + // Plain style is only selected for scalars without line breaks (analyze_scalar), so the + // only decision points are spaces; everything between them is copied as one run. + let mut spaces = false; + let mut run_start = 0; + for (index, c) in value.char_indices() { + if c != ' ' { + spaces = false; + continue; + } + self.put_run(&value[run_start..index]); + run_start = index + 1; + let next_is_space = value[index + 1..].starts_with(' '); + if allow_breaks && !spaces && self.column > BEST_WIDTH && !next_is_space { + self.write_indent(); + } else { + self.put(' '); + } + spaces = true; + } + self.put_run(&value[run_start..]); + self.whitespace = false; + self.indention = false; + } + + fn write_break(&mut self, c: char) { + if c == '\n' { + self.put_break(); + } else { + self.out.push(c); + self.column = 0; + } + } + + fn write_single_quoted(&mut self, value: &str, allow_breaks: bool) { + self.write_indicator("'", true, false, false); + let mut spaces = false; + let mut breaks = false; + let mut run_start = 0; + let mut run_open = false; // a pending plain run begins at run_start + for (index, c) in value.char_indices() { + let special = c == ' ' || c == '\'' || is_break(c); + if !special { + if !run_open { + run_start = index; + run_open = true; + if breaks { + self.write_indent(); + } + } + spaces = false; + breaks = false; + continue; + } + if run_open { + self.put_run(&value[run_start..index]); + run_open = false; + self.indention = false; + } + if c == ' ' { + let is_first = index == 0; + let is_last = index + 1 == value.len(); + let next_is_space = value[index + 1..].starts_with(' '); + if allow_breaks + && !spaces + && self.column > BEST_WIDTH + && !is_first + && !is_last + && !next_is_space + { + self.write_indent(); + } else { + self.put(c); + } + spaces = true; + } else if is_break(c) { + if !breaks && c == '\n' { + self.put_break(); + } + self.write_break(c); + self.indention = true; + breaks = true; + } else { + // a quote: written as two quotes, and it ends any break run like a plain char + if breaks { + self.write_indent(); + } + self.put('\''); + self.put('\''); + self.indention = false; + spaces = false; + breaks = false; + } + } + if run_open { + self.put_run(&value[run_start..]); + self.indention = false; + } + if breaks { + self.write_indent(); + } + self.write_indicator("'", false, false, false); + self.whitespace = false; + self.indention = false; + } + + fn needs_escape(&self, c: char) -> bool { + !is_printable(c) + || (!self.unicode && !c.is_ascii()) + || c == '\u{FEFF}' + || is_break(c) + || c == '"' + || c == '\\' + } + + fn write_double_quoted(&mut self, value: &str, allow_breaks: bool) { + self.write_indicator("\"", true, false, false); + let mut spaces = false; + let mut run_start = 0; + for (index, c) in value.char_indices() { + let escape = self.needs_escape(c); + if !escape && c != ' ' { + spaces = false; + continue; + } + self.put_run(&value[run_start..index]); + run_start = index + c.len_utf8(); + if escape { + self.write_escape(c); + spaces = false; + } else { + let is_first = index == 0; + let is_last = index + 1 == value.len(); + if allow_breaks && !spaces && self.column > BEST_WIDTH && !is_first && !is_last { + self.write_indent(); + if value[index + 1..].starts_with(' ') { + self.put('\\'); + } + } else { + self.put(c); + } + spaces = true; + } + } + self.put_run(&value[run_start..]); + self.write_indicator("\"", false, false, false); + self.whitespace = false; + self.indention = false; + } + + fn write_escape(&mut self, c: char) { + self.put('\\'); + let v = c as u32; + match v { + 0x00 => self.put('0'), + 0x07 => self.put('a'), + 0x08 => self.put('b'), + 0x09 => self.put('t'), + 0x0A => self.put('n'), + 0x0B => self.put('v'), + 0x0C => self.put('f'), + 0x0D => self.put('r'), + 0x1B => self.put('e'), + 0x22 => self.put('"'), + 0x5C => self.put('\\'), + 0x85 => self.put('N'), + 0xA0 => self.put('_'), + 0x2028 => self.put('L'), + 0x2029 => self.put('P'), + _ => { + let (marker, width) = if v <= 0xFF { + ('x', 2) + } else if v <= 0xFFFF { + ('u', 4) + } else { + ('U', 8) + }; + self.put(marker); + for shift in (0..width).rev() { + let digit = (v >> (shift * 4)) & 0x0F; + self.put(char::from_digit(digit, 16).unwrap().to_ascii_uppercase()); + } + } + } + } +} + +/// `SafeRepresenter.represent_float` text for a Python `repr(float)`. +fn float_text(repr: &str) -> String { + match repr { + "nan" => ".nan".to_string(), + "inf" => ".inf".to_string(), + "-inf" => "-.inf".to_string(), + _ => { + let mut value = repr.to_lowercase(); + if !value.contains('.') && value.contains('e') { + value = value.replacen('e', ".0e", 1); + } + value + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn map(entries: Vec<(&str, Node)>) -> Node { + Node::Map( + entries + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect(), + ) + } + + #[test] + fn sorts_keys_and_uses_indentless_sequences() { + let root = map(vec![ + ( + "b", + Node::Seq(vec![Node::Int("1".into()), Node::Str("x".into())]), + ), + ("a", map(vec![("k", Node::Bool(true))])), + ]); + let text = Emitter::dump(&root, false, 0).unwrap(); + assert_eq!(text, "a:\n k: true\nb:\n- 1\n- x\n"); + } + + #[test] + fn quotes_lookalike_strings_and_empty_collections() { + let root = map(vec![ + ("s", Node::Str("true".into())), + ("e", Node::Str(String::new())), + ("l", Node::Seq(vec![])), + ("m", Node::Map(vec![])), + ]); + let text = Emitter::dump(&root, false, 0).unwrap(); + assert_eq!(text, "e: ''\nl: []\nm: {}\ns: 'true'\n"); + } + + #[test] + fn escapes_non_ascii_unless_unicode_allowed() { + let root = map(vec![("k", Node::Str("zażółć".into()))]); + assert_eq!( + Emitter::dump(&root, false, 0).unwrap(), + "k: \"za\\u017C\\xF3\\u0142\\u0107\"\n" + ); + assert_eq!(Emitter::dump(&root, true, 0).unwrap(), "k: zażółć\n"); + } + + #[test] + fn detects_non_string_lookalikes() { + for v in [ + "", + "true", + "No", + "null", + "~", + "12", + "-3.5", + "0x1F", + "2026-10-19", + "=", + "<<", + "1.0e+3", + ".inf", + ] { + assert!(resolves_to_non_str(v), "{v:?} must be quoted"); + } + for v in [ + "hello", "PLF-001", "1.2.3", "nope", "tru", "10-19", "a b", "1e3", + ] { + assert!(!resolves_to_non_str(v), "{v:?} must stay plain"); + } + } + + #[test] + fn ascii_analysis_equals_general_analysis() { + let alphabet: Vec = " \n\r\t-?:#,[]{}&*!|>%@`'\"a.0=\u{7f}\u{1}" + .chars() + .collect(); + let mut stack: Vec = vec![String::new()]; + for _ in 0..4 { + let mut next = Vec::new(); + for prefix in &stack { + for c in &alphabet { + let mut candidate = prefix.clone(); + candidate.push(*c); + let a = analyze_ascii(candidate.as_bytes()); + let g = analyze_general(&candidate, false); + assert_eq!( + ( + a.multiline, + a.flow_plain_allowed, + a.block_plain_allowed, + a.single_quoted_allowed + ), + ( + g.multiline, + g.flow_plain_allowed, + g.block_plain_allowed, + g.single_quoted_allowed + ), + "{candidate:?}" + ); + next.push(candidate); + } + } + stack = next; + } + } +} diff --git a/packages/planfile-native/tests/test_parity.py b/packages/planfile-native/tests/test_parity.py new file mode 100644 index 0000000..800b123 --- /dev/null +++ b/packages/planfile-native/tests/test_parity.py @@ -0,0 +1,210 @@ +"""Differential tests: planfile_native must reproduce PyYAML/json byte for byte or refuse.""" + +from __future__ import annotations + +import datetime as dt +import json +import os +import random +import stat +import string + +import pytest +import yaml + +native = pytest.importorskip("planfile_native") +if not native.AVAILABLE: # pragma: no cover + pytest.skip("compiled extension missing", allow_module_level=True) + +try: + Dumper = yaml.CSafeDumper +except AttributeError: # pragma: no cover + pytest.skip("PyYAML without libyaml: reference output differs", allow_module_level=True) + + +def reference(data, allow_unicode=False): + return yaml.dump(data, default_flow_style=False, allow_unicode=allow_unicode, Dumper=Dumper) + + +CURATED = [ + {"a": 1}, + {}, + [], + [1, 2, [3, [4, {"k": []}]], {}], + {"s": ["", " ", " x", "x ", " x ", "a b", "a\tb", "tab\t", "multi\nline", "trail\n", "\nlead", + "a\n b", "a \nb", "x\r\ny", "'q'", '"d"', "back\\slash", "#c", "a #c", "a: b", "a:b", + "- x", "-x", "-", "?", "? x", ":x", "key: v", "[x]", "{x}", "x,y", "&a", "*a", "!t", + "|", ">", "%", "@", "`", "---", "...", "--- x", "null", "Null", "~", "true", "Yes", "off", + "1", "01", "0o7", "0x1f", "1_000", "1.5", "1e3", "1.0e+3", ".5", ".inf", "-.INF", ".nan", + "2026-10-19", "2026-10-19 12:00:00", "12:30:45", "=", "<<", "!", "&", "*"]}, + {"unicode": ["zażółć gęślą jaźń", "naïve café", "日本語", "emoji 😀 x", "\u0085nel", "\u2028ls", + "\u00a0nbsp", "\ufeffbom", "\x00nul", "\x1b[0m", "\x7fdel", "\ud7ff\ue000\ufffd"]}, + {"long": "word " * 60, "longq": "it's " * 40, "longd": "tab\t" + "word " * 60, "x" * 200: 1, + "nl\nkey": 2, "k" * 128: 3, "k" * 129: 4}, + {"nested": {"deep": {"deeper": {"list": [{"a": 1, "b": [1, 2]}, [[1], [2, [3]]]]}}}}, + {"num": [0, -1, 10**30, 1.0, -2.5, 1e-05, 1e16, 1.5e300, float("inf"), float("-inf"), True, False, None]}, + {"when": dt.datetime(2026, 10, 19, 16, 38, 31), "tz": dt.datetime(2026, 10, 19, 16, 38, 31, 123456, + tzinfo=dt.timezone.utc), + "day": dt.date(2026, 10, 19)}, + {"tuple": (1, "a", (2,))}, + {"z": 1, "a": {"y": 1, "b": 2}, "Z": 3, "_": 4, "é": 5}, +] + + +@pytest.mark.parametrize("allow_unicode", [False, True]) +@pytest.mark.parametrize("data", CURATED, ids=lambda d: repr(d)[:40]) +def test_curated_yaml_matches_pyyaml(data, allow_unicode): + assert native.dump_yaml(data, allow_unicode) == reference(data, allow_unicode) + + +ALPHABET = ( + string.ascii_letters + string.digits + " " * 6 + "\n\t'\"\\:#-?,[]{}&*!|>%@`~=<.\r" + + "żółćéß日\u0085\u2028\u00a0\u2029" +) +WORDS = ["null", "true", "no", "1", "0", "-", "yes", "", "~", "<<", "2026-01-02", "1.5", "a b", "---", "..."] + + +def random_string(rng: random.Random) -> str: + if rng.random() < 0.15: + return rng.choice(WORDS) + base = "".join(rng.choice(ALPHABET) for _ in range(rng.choice([1, 2, 3, 5, 12, 40, 90, 200]))) + if rng.random() < 0.2: + base = " ".join(["word"] * rng.randint(10, 60)) + return base + + +def random_value(rng: random.Random, depth: int): + kind = rng.random() + if depth > 4 or kind < 0.45: + return rng.choice([ + lambda: random_string(rng), lambda: rng.randint(-10**12, 10**12), + lambda: rng.uniform(-1e6, 1e6), lambda: rng.choice([True, False, None]), + lambda: rng.choice([1e-7, 1e22, 0.1, 1 / 3]), + ])() + if kind < 0.7: + return [random_value(rng, depth + 1) for _ in range(rng.randint(0, 5))] + return {random_string(rng) if rng.random() < 0.8 else f"k{i}": random_value(rng, depth + 1) + for i in range(rng.randint(0, 6))} + + +@pytest.mark.parametrize("seed", range(40)) +@pytest.mark.parametrize("allow_unicode", [False, True]) +def test_random_documents_match_pyyaml(seed, allow_unicode): + rng = random.Random(seed) + for _ in range(60): + data = {"root": random_value(rng, 0)} if rng.random() < 0.5 else [random_value(rng, 0)] + try: + expected = reference(data, allow_unicode) + except Exception: + with pytest.raises(Exception): + native.dump_yaml(data, allow_unicode) + continue + assert native.dump_yaml(data, allow_unicode) == expected, repr(data) + assert yaml.safe_load(native.dump_yaml(data, allow_unicode)) == yaml.safe_load(expected) + + +class StrSub(str): + pass + + +@pytest.mark.parametrize( + "data", + [ + {"a": StrSub("x")}, + {"a": {1, 2}}, + {1: "int key"}, + {"a": b"bytes"}, + {"a": datetime_time} if (datetime_time := dt.time(1, 2)) else None, + "scalar root", + ], +) +def test_unsupported_values_raise_unsupported(data): + with pytest.raises(native.Unsupported): + native.dump_yaml(data) + + +def test_shared_container_is_refused_because_pyyaml_emits_anchors(): + shared = [1] + data = {"a": shared, "b": shared} + assert "&id001" in reference(data) + with pytest.raises(native.Unsupported): + native.dump_yaml(data) + + +def test_cycle_is_refused(): + data: dict = {} + data["self"] = data + with pytest.raises(native.Unsupported): + native.dump_yaml(data) + + +def test_lone_surrogate_is_refused(): + with pytest.raises(native.Unsupported): + native.dump_yaml({"a": "\ud800"}) + + +def _json_safe(value): + if isinstance(value, dict): + return {_json_safe(k): _json_safe(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_json_safe(v) for v in value] + if isinstance(value, (dt.datetime, dt.date, dt.time)): + return value.isoformat() + return value + + +JSON_CASES = [d for d in CURATED if not (isinstance(d, dict) and any(not isinstance(k, str) for k in d))] + + +@pytest.mark.parametrize("data", JSON_CASES, ids=lambda d: repr(d)[:40]) +def test_json_matches_cpython_encoder(data): + assert native.dump_json(data) == json.dumps(_json_safe(data), ensure_ascii=False) + + +@pytest.mark.parametrize("data", CURATED + [dt.time(3, 4)], ids=lambda d: repr(d)[:40]) +def test_normalize_matches_json_safe(data): + assert native.normalize(data) == _json_safe(data) + assert type(native.normalize(data)) is type(_json_safe(data)) + + +def test_write_sprint_writes_yaml_and_matching_mirror(tmp_path): + path = tmp_path / "sub" / "current.yaml" + data = {"tickets": {"PLF-1": {"name": "zażółć", "when": dt.datetime(2026, 1, 2, 3, 4, 5)}}} + mtime, mirror_error = native.write_sprint(str(path), data, True, 1, ".fast.json", True) + assert mirror_error is None + assert path.read_text(encoding="utf-8") == reference(data, True) + assert mtime == path.stat().st_mtime_ns + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + mirror = json.loads((tmp_path / "sub" / "current.yaml.fast.json").read_text(encoding="utf-8")) + assert mirror == {"version": 1, "yaml_mtime_ns": mtime, "data": _json_safe(data)} + assert [p.name for p in path.parent.iterdir() if p.name.endswith(".tmp")] == [] + + +def test_write_sprint_unsupported_leaves_filesystem_untouched(tmp_path): + path = tmp_path / "current.yaml" + with pytest.raises(native.Unsupported): + native.write_sprint(str(path), {"a": {1, 2}}, True, 1, ".fast.json", False) + assert list(tmp_path.iterdir()) == [] + + +def test_write_sprint_reports_mirror_failure_without_raising(tmp_path): + path = tmp_path / "current.yaml" + (tmp_path / "current.yaml.fast.json").mkdir() # rename onto a directory fails + mtime, mirror_error = native.write_sprint(str(path), {"a": 1}, True, 1, ".fast.json", False) + assert mirror_error + assert path.read_text() == "a: 1\n" and mtime == path.stat().st_mtime_ns + + +def test_write_sprint_raises_oserror_for_unwritable_target(tmp_path): + blocker = tmp_path / "file" + blocker.write_text("x") + with pytest.raises(OSError): + native.write_sprint(str(blocker / "child.yaml"), {"a": 1}, True, 1, ".fast.json", False) + + +def test_atomic_write_text_replaces_content(tmp_path): + target = tmp_path / "a.txt" + native.atomic_write_text(str(target), "one") + native.atomic_write_text(str(target), "dwa żółw", False) + assert target.read_text(encoding="utf-8") == "dwa żółw" + assert os.listdir(tmp_path) == ["a.txt"] diff --git a/planfile/core/fastio.py b/planfile/core/fastio.py index 09b9279..686c458 100644 --- a/planfile/core/fastio.py +++ b/planfile/core/fastio.py @@ -72,11 +72,20 @@ def _json_safe(value: Any) -> Any: return value +def _normalize(value: Any) -> Any: + """``_json_safe`` through the native kernel when active (same result, same types).""" + from planfile.core import native + + normalized = native.normalize(value) + return _json_safe(value) if normalized is None else normalized + + def load_yaml_text(text: str) -> Any: - return _json_safe(yaml.load(text, Loader=FastLoader)) + return _normalize(yaml.load(text, Loader=FastLoader)) -def dump_yaml(data: Any, *, allow_unicode: bool = False) -> str: +def dump_yaml_python(data: Any, *, allow_unicode: bool = False) -> str: + """Reference serializer; the native kernel must reproduce this output byte for byte.""" return yaml.dump( data, default_flow_style=False, @@ -85,6 +94,15 @@ def dump_yaml(data: Any, *, allow_unicode: bool = False) -> str: ) +def dump_yaml(data: Any, *, allow_unicode: bool = False) -> str: + from planfile.core import native + + text = native.dump_yaml(data, allow_unicode=allow_unicode) + if text is not None: + return text + return dump_yaml_python(data, allow_unicode=allow_unicode) + + def _stat_mtime_ns(path: Path) -> int | None: try: return path.stat().st_mtime_ns @@ -130,7 +148,7 @@ def write_mirror(yaml_path: Path, data: Any, *, mtime_ns: int | None = None) -> payload = { "version": _MIRROR_VERSION, "yaml_mtime_ns": mtime_ns, - "data": _json_safe(data), + "data": _normalize(data), } try: _atomic_write_text(mirror_path(yaml_path), json.dumps(payload, ensure_ascii=False)) @@ -138,12 +156,40 @@ def write_mirror(yaml_path: Path, data: Any, *, mtime_ns: int | None = None) -> # The mirror is an optimization only; a failed write must never break # a ticket mutation. It must remain observable, otherwise a permanent # full-YAML fallback looks like unexplained API latency. - warnings.warn( - f"planfile: JSON mirror for {yaml_path.name} not written " - f"({type(exc).__name__}: {exc}); reads fall back to full YAML parsing", - RuntimeWarning, - stacklevel=2, - ) + _warn_mirror_failure(yaml_path, f"{type(exc).__name__}: {exc}", stacklevel=3) + + +def _warn_mirror_failure(yaml_path: Path, reason: str, *, stacklevel: int = 2) -> None: + warnings.warn( + f"planfile: JSON mirror for {yaml_path.name} not written " + f"({reason}); reads fall back to full YAML parsing", + RuntimeWarning, + stacklevel=stacklevel, + ) + + +def write_sprint_native(yaml_path: Path, data: Any, *, allow_unicode: bool) -> bool: + """Write ``yaml_path`` and its mirror with the native kernel. + + Returns False without touching the filesystem when the kernel is disabled, missing or + refuses the value; the caller must then use the Python writer. The mirror is stamped with the + mtime the kernel read right after its own rename, as ``Store._write_yaml_atomic`` does. + """ + from planfile.core import native + + outcome = native.write_sprint( + yaml_path, + data, + allow_unicode=allow_unicode, + mirror_version=_MIRROR_VERSION, + mirror_suffix=_MIRROR_SUFFIX, + ) + if outcome is None: + return False + _mtime_ns, mirror_error = outcome + if mirror_error is not None: + _warn_mirror_failure(yaml_path, f"OSError: {mirror_error}", stacklevel=3) + return True def read_mirror(yaml_path: Path, yaml_mtime_ns: int) -> Any | None: diff --git a/planfile/core/native.py b/planfile/core/native.py new file mode 100644 index 0000000..f6d0274 --- /dev/null +++ b/planfile/core/native.py @@ -0,0 +1,104 @@ +"""Optional Rust storage kernel (``planfile-native``) behind a fail-closed switch. + +Every helper returns ``None`` when the native path is disabled, not installed, or refuses the +value (``planfile_native.Unsupported``). Callers then run their pure-Python implementation, so +the result is always the one PyYAML would have produced. + +``PLANFILE_NATIVE`` + ``auto`` (default) use the extension when importable; ``off`` never use it; ``verify`` use it + and compare every YAML document with PyYAML's, raising ``RuntimeError`` on a mismatch. +``PLANFILE_NATIVE_FSYNC`` + ``0``/``false``/``off``/``no`` skips fsync of native writes; any other value keeps it. +""" + +from __future__ import annotations + +import os +from typing import Any + +try: + import planfile_native as _rs + + _AVAILABLE = bool(_rs.AVAILABLE) +except ImportError: # pragma: no cover - extension is optional + _rs = None + _AVAILABLE = False + +MODE_ENV = "PLANFILE_NATIVE" +FSYNC_ENV = "PLANFILE_NATIVE_FSYNC" +_FALSE_WORDS = frozenset({"0", "false", "off", "no"}) +_MODES = frozenset({"auto", "off", "verify"}) + + +def mode() -> str: + """Effective mode; unknown values fail closed to ``off``.""" + value = os.environ.get(MODE_ENV, "auto").strip().lower() or "auto" + return value if value in _MODES else "off" + + +def available() -> bool: + return _AVAILABLE + + +def enabled() -> bool: + return _AVAILABLE and mode() != "off" + + +def fsync_enabled() -> bool: + return os.environ.get(FSYNC_ENV, "1").strip().lower() not in _FALSE_WORDS + + +def _verify(data: Any, allow_unicode: bool, native_text: str) -> None: + from planfile.core.fastio import dump_yaml_python + + expected = dump_yaml_python(data, allow_unicode=allow_unicode) + if expected != native_text: + raise RuntimeError( + "planfile-native YAML output differs from PyYAML; set PLANFILE_NATIVE=off " + "and report this document shape" + ) + + +def dump_yaml(data: Any, *, allow_unicode: bool = False) -> str | None: + if not enabled(): + return None + try: + text = _rs.dump_yaml(data, allow_unicode) + except _rs.Unsupported: + return None + if mode() == "verify": + _verify(data, allow_unicode, text) + return text + + +def normalize(value: Any) -> Any | None: + """``fastio._json_safe`` result, or ``None`` when the native path is not in use.""" + if not enabled(): + return None + return _rs.normalize(value) + + +def write_sprint( + path: Any, + data: Any, + *, + allow_unicode: bool, + mirror_version: int, + mirror_suffix: str, +) -> tuple[int, str | None] | None: + """Write YAML + JSON mirror natively; ``None`` means "use the Python writer".""" + if not enabled(): + return None + if mode() == "verify" and dump_yaml(data, allow_unicode=allow_unicode) is None: + return None + try: + return _rs.write_sprint( + os.fspath(path), + data, + allow_unicode, + mirror_version, + mirror_suffix, + fsync_enabled(), + ) + except _rs.Unsupported: + return None diff --git a/planfile/core/store.py b/planfile/core/store.py index fc2363b..9e1166f 100644 --- a/planfile/core/store.py +++ b/planfile/core/store.py @@ -746,7 +746,13 @@ def mutation_lock(self, *, timeout_seconds: float | None = None): self._invalidate_ticket_index_signature_cache() def _write_yaml_atomic(self, path: Path, data: dict, *, allow_unicode: bool = False) -> None: - from planfile.core.fastio import dump_yaml, write_mirror + from planfile.core.fastio import dump_yaml, write_mirror, write_sprint_native + + # Native kernel: one call renders YAML + mirror and writes both atomically. It returns + # False (touching nothing) whenever PyYAML must handle the value, so the path below + # remains the reference implementation. + if write_sprint_native(path, data, allow_unicode=allow_unicode): + return path.parent.mkdir(parents=True, exist_ok=True) content = dump_yaml(data, allow_unicode=allow_unicode) diff --git a/project/TICKETS.md b/project/TICKETS.md index 4c8e12c..028b0dc 100644 --- a/project/TICKETS.md +++ b/project/TICKETS.md @@ -128,4 +128,5 @@ This file indexes governance tickets without taking ownership of | **ticket-216** | [`README.md`](./ticket-216/README.md) | - | - | - | - | - | | **ticket-217** | [`README.md`](./ticket-217/README.md) | - | - | - | - | - | | **ticket-218** | [`README.md`](./ticket-218/README.md) | - | - | - | - | - | +| **ticket-220** | [`README.md`](./ticket-220/README.md) | - | - | - | - | - | diff --git a/project/ticket-226/README.md b/project/ticket-226/README.md new file mode 100644 index 0000000..b1aedfe --- /dev/null +++ b/project/ticket-226/README.md @@ -0,0 +1,72 @@ +# Ticket 226: Extract storage bottlenecks into reusable Rust package planfile-native + +- **ID**: ticket-226 +- **Owner**: agent:antigravity (requested by the repository owner) +- **Status**: IN_PROGRESS +- **Workflow state**: EDIT +- **Created**: 2026-10-19 + +## Goal and scope + +Move the measured storage bottleneck of planfile into a reusable Rust package that planfile +imports optionally, keeping YAML as the single source of truth and the on-disk format unchanged. + +Decisions confirmed by the owner: new crate inside this repository +(`packages/planfile-native`), scope = write path (YAML dump, JSON mirror, atomic write) plus +mirror normalization, YAML output byte-identical to PyYAML `CSafeDumper` with a Python fallback, +data-plane/data-gateway handled as an aligned storage-kernel contract (no event-sourcing +migration). + +Baseline profile (3000 tickets, one 7.9 MB `current.yaml`): every mutation spent ~95% of its time +in `yaml.dump` (pure Python representer over ~340k nodes) plus the `_json_safe` walk and +`json.dumps` for the mirror. The existing `planfile-io` wheel is not a usable substitute: +`fastio.read_yaml_native` is dead code, its mirror schema (`mtime_ns`) is incompatible with +planfile's (`yaml_mtime_ns`), and its `serde_yaml` output formats differently from PyYAML. + +## Acceptance criteria + +- [x] AC-01: `planfile_native.dump_yaml` is byte-identical to `yaml.dump(..., CSafeDumper, + default_flow_style=False)` for curated edge cases and 4800 random documents, with and without + `allow_unicode` (`packages/planfile-native/tests/test_parity.py`). +- [x] AC-02: Unsupported values (subclasses, sets, non-str keys, shared containers that PyYAML + anchors, cycles, lone surrogates) raise `Unsupported` before any file is touched; planfile then + uses PyYAML (`tests/test_native_integration.py`). +- [x] AC-03: `PLANFILE_NATIVE=off|auto|verify` and `PLANFILE_NATIVE_FSYNC` switch behavior at + runtime; unknown values fail closed to `off`; no policy is hardcoded in the crate. +- [x] AC-04: Full planfile suite passes with `PLANFILE_NATIVE=verify` (every write compared with + PyYAML): 1292 passed, 6 skipped. `tests/test_release_availability.py::test_release_metadata_ + matches_published_offline_snapshot` fails identically on an unmodified tree and is unrelated. +- [x] AC-05: Same-workload benchmark (`packages/planfile-native/benches/bench_store.py 3000`, + 2 runs x 7 samples each, medians): + + | operation | PLANFILE_NATIVE=off | auto | speed-up | + |---|---|---|---| + | update_ticket | 695 ms | 162 ms | 4.3x | + | create_ticket | 678 ms | 148 ms | 4.6x | + | list_tickets (warm) | 176 ms | 139 ms | 1.3x | + | get_ticket (warm) | 34 ms | 24 ms | 1.4x | + | list_tickets (cold, no mirror) | 1014 ms | 871 ms | 1.2x | + +- [ ] AC-06: Governance gates (`./project/governance-check.sh`, `wellman check`) pass. + +## Remaining bottlenecks (follow-up, not in this ticket) + +- Each mutation still `json.loads` the 8 MB mirror about twice (~40 ms) and rebuilds index and + evidence state in Python; a byte-offset index would let `get_ticket` avoid parsing a whole + sprint. +- Per-ticket projection (`_project_ticket_evidence`, Pydantic validation) dominates warm + `list_tickets`. +- Cold YAML parse remains PyYAML/libyaml; a native parser must reproduce YAML 1.1 resolution + (`yes`/`no`, sexagesimal ints, timestamps) before it can replace it safely. +- A `docs/` performance report and a `wellmanifest/performance` plan record belong to the + integration workstream (`docs/**`), which this application ticket does not own. + +## Rollback + +Set `PLANFILE_NATIVE=off` (no data migration is involved because files are byte-identical), or +revert this ticket's commit. + +## Tracking boundary + +This directory contains the minimal reviewed intent. Optional participant prose and raw command +logs are not required delivery output. diff --git a/project/ticket-226/intent.json b/project/ticket-226/intent.json new file mode 100644 index 0000000..86730d9 --- /dev/null +++ b/project/ticket-226/intent.json @@ -0,0 +1,28 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-226", + "summary": "Extract storage bottlenecks into reusable Rust package planfile-native", + "workstream": "application", + "classification": { + "kind": "SERVICE", + "priority": "P2", + "origin": "requested" + }, + "allowedPaths": [ + "project/ticket-226/**", + "TODO.md", + "project/TICKETS.md", + "packages/planfile-native/**", + "planfile/core/fastio.py", + "planfile/core/native.py", + "planfile/core/store.py", + "tests/test_native_*.py" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/tests/test_native_integration.py b/tests/test_native_integration.py new file mode 100644 index 0000000..d2bb253 --- /dev/null +++ b/tests/test_native_integration.py @@ -0,0 +1,129 @@ +"""planfile wiring of the optional planfile-native kernel (switch, fallback, parity).""" + +from __future__ import annotations + +import json +import warnings + +import pytest +import yaml + +from planfile.core import fastio, native +from planfile.core.models import Ticket +from planfile.core.store import Store + +needs_native = pytest.mark.skipif(not native.available(), reason="planfile-native not installed") + + +def _store(tmp_path) -> Store: + store = Store(tmp_path) + store.init() + return store + + +def _populate(store: Store) -> None: + store.create_tickets_bulk( + [ + Ticket(id=f"PLF-{i:03d}", name=f"Zażółć {i}: true", description="word " * 60 + f"\nline {i}") + for i in range(25) + ] + ) + store.update_ticket("PLF-003", name="renamed 'quoted'") + store.delete_ticket("PLF-004") + + +def test_mode_parsing_fails_closed(monkeypatch): + monkeypatch.setenv(native.MODE_ENV, "bogus") + assert native.mode() == "off" and not native.enabled() + monkeypatch.setenv(native.MODE_ENV, " VERIFY ") + assert native.mode() == "verify" + monkeypatch.delenv(native.MODE_ENV) + assert native.mode() == "auto" + + +@pytest.mark.parametrize("value,expected", [("1", True), ("0", False), ("off", False), ("No", False), ("yes", True)]) +def test_fsync_switch(monkeypatch, value, expected): + monkeypatch.setenv(native.FSYNC_ENV, value) + assert native.fsync_enabled() is expected + + +def test_off_mode_never_calls_the_kernel(monkeypatch, tmp_path): + monkeypatch.setenv(native.MODE_ENV, "off") + assert native.dump_yaml({"a": 1}) is None + assert native.normalize({"a": 1}) is None + assert native.write_sprint(tmp_path / "x.yaml", {"a": 1}, allow_unicode=True, mirror_version=1, mirror_suffix=".m") is None + assert not (tmp_path / "x.yaml").exists() + + +@needs_native +def test_native_written_sprint_files_are_pyyaml_fixed_points(monkeypatch, tmp_path): + """Wall-clock event hashes make two runs differ, so check the invariant instead: re-dumping + what the kernel wrote with PyYAML must reproduce the file byte for byte.""" + monkeypatch.setenv(native.MODE_ENV, "auto") + _populate(_store(tmp_path)) + files = sorted((tmp_path / ".planfile" / "sprints").rglob("*.yaml")) + assert files + for path in files: + text = path.read_text(encoding="utf-8") + assert fastio.dump_yaml_python(yaml.safe_load(text), allow_unicode=True) == text, path.name + + +@needs_native +def test_native_write_leaves_a_valid_mirror_and_reads_back(monkeypatch, tmp_path): + monkeypatch.setenv(native.MODE_ENV, "auto") + store = _store(tmp_path) + _populate(store) + current = tmp_path / ".planfile" / "sprints" / "current.yaml" + mirror = json.loads(fastio.mirror_path(current).read_text(encoding="utf-8")) + assert mirror["version"] == fastio._MIRROR_VERSION + assert mirror["yaml_mtime_ns"] == current.stat().st_mtime_ns + assert fastio.read_mirror(current, current.stat().st_mtime_ns) == fastio.load_yaml_text(current.read_text(encoding="utf-8")) + assert store.get_ticket("PLF-003").name == "renamed 'quoted'" + assert store.get_ticket("PLF-004") is None + + +@needs_native +def test_unsupported_value_falls_back_to_pyyaml(monkeypatch, tmp_path): + monkeypatch.setenv(native.MODE_ENV, "auto") + shared = ["x"] + data = {"a": shared, "b": shared} # PyYAML emits an anchor; the kernel must refuse + target = tmp_path / "shared.yaml" + assert not fastio.write_sprint_native(target, data, allow_unicode=True) + assert not target.exists() + store = _store(tmp_path) + store._write_yaml_atomic(target, data, allow_unicode=True) + assert "&id001" in target.read_text(encoding="utf-8") + + +@needs_native +def test_verify_mode_accepts_matching_output_and_rejects_a_mismatch(monkeypatch, tmp_path): + monkeypatch.setenv(native.MODE_ENV, "verify") + assert fastio.dump_yaml({"k": "v"}) == "k: v\n" + monkeypatch.setattr(fastio, "dump_yaml_python", lambda data, allow_unicode=False: "different\n") + with pytest.raises(RuntimeError, match="differs from PyYAML"): + native.dump_yaml({"k": "v"}) + + +@needs_native +def test_mirror_write_failure_warns_without_raising(monkeypatch, tmp_path): + monkeypatch.setenv(native.MODE_ENV, "auto") + target = tmp_path / "current.yaml" + fastio.mirror_path(target).mkdir() # rename onto a directory fails + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + assert fastio.write_sprint_native(target, {"a": 1}, allow_unicode=True) + assert target.read_text() == "a: 1\n" + assert any("JSON mirror for current.yaml not written" in str(w.message) for w in caught) + + +@needs_native +def test_load_yaml_text_normalizes_temporal_values_identically(monkeypatch): + text = "when: 2026-10-19 16:38:31\nday: 2026-10-19\nitems: [1, 2]\n" + monkeypatch.setenv(native.MODE_ENV, "off") + expected = fastio.load_yaml_text(text) + monkeypatch.setenv(native.MODE_ENV, "auto") + assert fastio.load_yaml_text(text) == expected == { + "when": "2026-10-19T16:38:31", + "day": "2026-10-19", + "items": [1, 2], + } From c238319de9ee1d07d20fa4897ba1f9b823d77ab6 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Mon, 19 Oct 2026 17:27:14 +0200 Subject: [PATCH 2/2] test(native): use skipif instead of pytest.skip in parity tests (ticket-226) --- packages/planfile-native/tests/test_parity.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/planfile-native/tests/test_parity.py b/packages/planfile-native/tests/test_parity.py index 800b123..ec1c9d8 100644 --- a/packages/planfile-native/tests/test_parity.py +++ b/packages/planfile-native/tests/test_parity.py @@ -13,13 +13,13 @@ import yaml native = pytest.importorskip("planfile_native") -if not native.AVAILABLE: # pragma: no cover - pytest.skip("compiled extension missing", allow_module_level=True) -try: - Dumper = yaml.CSafeDumper -except AttributeError: # pragma: no cover - pytest.skip("PyYAML without libyaml: reference output differs", allow_module_level=True) +# The reference output is libyaml's; PyYAML's pure-Python dumper formats a few cases differently. +Dumper = getattr(yaml, "CSafeDumper", yaml.SafeDumper) +pytestmark = pytest.mark.skipif( + not (native.AVAILABLE and hasattr(yaml, "CSafeDumper")), + reason="needs the compiled extension and PyYAML built with libyaml", +) def reference(data, allow_unicode=False):