diff --git a/README.md b/README.md index e0b66e8..49a9c5d 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,61 @@ -# Extended Data Workspace +# Extended Data ![Extended Data — structured values crossing clear data boundaries](https://raw.githubusercontent.com/jbcom/extended-data/main/docs/assets/extended-data-hero.png) -This repository is a `uv` workspace for the Extended Data package family. +Extended Data is a Python package family for moving structured values across +clear boundaries: parsing input, normalizing and transforming it, preserving +ergonomic container behavior, and exporting a plain built-in value when it is +time to write or hand data to another library. + +Start with the [documentation site](https://extended-data.dev), especially the +[Getting Started](https://extended-data.dev/guides/getting-started.html) and +[Package Surface](https://extended-data.dev/guides/package-surface.html) guides. + +## Install and choose a layer + +```bash +pip install extended-data +``` + +Most application code needs only these three choices: + +1. Use `extended_data.primitives` for a single deterministic conversion, + serialization operation, transform, or redaction step. +2. Use `ExtendedData(value)` at an uncertain data boundary. It returns the + right extended shape (`ExtendedDict`, `ExtendedList`, `ExtendedString`, and + so on) while retaining normal Python collection behavior. +3. Use `DataFile` or `DataWorkflow` when reading, merging, transforming, or + writing a structured artifact is the actual unit of work. + +```python +from extended_data import DataWorkflow, ExtendedData +from extended_data.primitives import decode_json, encode_yaml + +incoming = decode_json('{"service": {"name": "api"}}') +config = ExtendedData(incoming).merge({"replicas": 2}) + +result = ( + DataWorkflow.from_value(config) + .transform("unhump") + .result() +) + +assert result.as_extended()["service"]["name"] == "api" +assert "replicas: 2" in encode_yaml(result.as_builtin()) +``` + +`ExtendedData` promotes nested values as they enter or mutate containers. Use +`as_builtin()` (or `to_builtin()`) at an explicit export boundary when an API, +serializer, or third-party library needs ordinary Python `dict`, `list`, +`str`, and scalar values. + +## Consumers that generate or operate code + +Automated and agentic consumers should use the same public contract as human +callers: import pure functions from `extended_data.primitives`, promote unknown +payloads with `ExtendedData`, and lower values only at explicit boundaries. +The [agentic consumer guide](https://extended-data.dev/guides/agentic-consumers.html) +sets out the safe integration rules, test plugin, and package ownership limits. ## Packages @@ -11,14 +64,17 @@ This repository is a `uv` workspace for the Extended Data package family. | `extended-data` | `packages/extended-data` | Runtime data primitives, containers, IO, workflows, inputs, logging, docs, and CLI | | `pytest-extended-data` | `packages/pytest-extended-data` | Reusable pytest fixtures and assertion helpers for Extended Data consumers | -The workspace root is not a published Python distribution. +The workspace root is not a published Python distribution. Install a package, +not the repository root, in downstream applications. ## Common Commands ```bash uv sync --all-packages --all-extras --dev tox -e lint,typecheck,audit,py311,py312,py313,py314,examples,docs,build +pnpm docs:validate ``` -The Sourcey documentation site lives in `docs/` and deploys to -. It is separate from the Python `uv` workspace. +The Sourcey documentation source lives in `docs/` and deploys to +. It is independently locked from the Python `uv` +workspace so documentation builds remain deterministic. diff --git a/docs/guides/agentic-consumers.md b/docs/guides/agentic-consumers.md new file mode 100644 index 0000000..c8d4494 --- /dev/null +++ b/docs/guides/agentic-consumers.md @@ -0,0 +1,79 @@ +# Agentic Consumers + +`extended-data` is useful to code-generating tools and agent runtimes for the +same reason it is useful to application code: it makes a data boundary +explicit. It is not an agent framework, a vendor SDK, a credential manager, or +a place to put provider-specific control flow. + +## Use the public data contract + +Use the smallest layer that completes the task: + +1. Import one pure operation from `extended_data.primitives` for parsing, + coercion, normalization, encoding, matching, or redaction. +2. Promote uncertain structured input with `ExtendedData(value)` when the + shape can be a mapping, sequence, or scalar. +3. Use `DataFile` or `DataWorkflow` for a named file or transformation + boundary. Keep the result as extended data until the caller explicitly + needs built-ins. + +```python +from extended_data import ExtendedData +from extended_data.primitives import decode_json, redact_sensitive_data + +raw = decode_json('{"service": {"name": "api"}, "token": "do-not-log"}') +payload = ExtendedData(raw).merge({"replicas": 2}) +safe_for_diagnostics = redact_sensitive_data(payload.as_builtin()) + +assert payload["service"]["name"].upper_first() == "Api" +assert safe_for_diagnostics["token"] == "[REDACTED]" +``` + +## Preserve the boundary + +Do not flatten a value merely to call a method. Nested literals are promoted +when they enter Extended Data containers, so normal indexing and mutation keep +the Tier 2 methods available. Conversely, call `as_builtin()` or `to_builtin()` +only when crossing into a serializer, an external API client, or a library +whose contract specifically requires built-in Python values. + +Treat redaction as a diagnostic boundary, not as a substitute for access +control. Redact before a value is logged, displayed, or attached to an error; +do not place credentials in prompts, source files, fixtures, or workflow +metadata. + +## Keep ownership clear + +This package owns data mechanics only. It must not acquire provider SDKs, +vendor API clients, secret-sync behavior, MCP adapters, agent runtime state, or +framework-specific tools. Those integrations belong to `vendor-fabric` and +`agentic-fabric`; they should consume this package's public data contract +instead of extending its internals. + +Do not restore removed compatibility imports such as `extended_data.connectors` +or `extended_data.secrets`. A failed import is an intentional migration signal. + +## Test downstream integrations + +Install `pytest-extended-data` in a consumer's test environment to get a small +representative payload, the polymorphic factory, and assertions for the plain +export boundary: + +```bash +uv add --dev pytest-extended-data +``` + +```python +from pytest_extended_data import assert_builtin_round_trip + + +def test_export_contract(extended_data_value): + assert_builtin_round_trip( + extended_data_value, + {"service": {"name": "api", "ports": [8080, 8443]}, "enabled": True}, + ) +``` + +Before generating a broad integration, consult the [Package Surface](package-surface.md), +[containers guide](../core/containers.md), and generated [API reference](../reference/index.md). +They are the source of truth for stable imports and behavior. diff --git a/docs/sourcey.config.ts b/docs/sourcey.config.ts index 733e378..0cb86c6 100644 --- a/docs/sourcey.config.ts +++ b/docs/sourcey.config.ts @@ -8,7 +8,7 @@ const generatedApiPages = readdirSync(join(import.meta.dirname, "reference/gener .sort(); const pages = { - gettingStarted: ["guides/getting-started", "guides/package-surface"], + gettingStarted: ["guides/getting-started", "guides/package-surface", "guides/agentic-consumers"], core: ["core/primitives", "core/containers", "core/workflows"], operations: ["operations/inputs", "operations/logging"], examples: ["examples/core", "examples/inputs", "examples/logging"], diff --git a/packages/extended-data/pyproject.toml b/packages/extended-data/pyproject.toml index 6d58aea..ccfe308 100644 --- a/packages/extended-data/pyproject.toml +++ b/packages/extended-data/pyproject.toml @@ -33,7 +33,7 @@ classifiers = [ ] dependencies = [ "deepmerge>=2.0", - "gitpython>=3.1.54", + "gitpython>=3.1.58", "inflection>=0.5.1", "num2words>=0.5.14", "orjson>=3.10.7", diff --git a/packages/extended-data/src/extended_data/primitives/types.py b/packages/extended-data/src/extended_data/primitives/types.py index 477e198..c49c658 100644 --- a/packages/extended-data/src/extended_data/primitives/types.py +++ b/packages/extended-data/src/extended_data/primitives/types.py @@ -22,7 +22,6 @@ - DATE_PATTERN: Regex for matching ISO 8601 date strings. - DATETIME_PATTERN: Regex for matching ISO 8601 datetime strings. - TIME_PATTERN: Regex for matching time strings. - - PATH_PATTERN: Regex for matching Unix and Windows-style paths. - INTEGER_PATTERN: Regex for matching integer strings. - NUMBER_PATTERN: Regex for matching numeric strings. - TRUTHY_PATTERN: Regex for matching truthy strings. @@ -54,13 +53,38 @@ r"^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?$" ) # Matches extended datetime formats like YYYY-MM-DDTHH:MM[:SS][.fff][Z|±hh:mm] TIME_PATTERN: re.Pattern[str] = re.compile(r"^\d{2}:\d{2}(:\d{2}(\.\d{1,6})?)?$") # Matches HH:MM[:SS] and microseconds -PATH_PATTERN: re.Pattern[str] = re.compile(r'^(?:[a-zA-Z]:)?[\\/](?:[^<>:"|?*\n]+[\\/])*[^<>:"|?*\n]*$') INTEGER_PATTERN: re.Pattern[str] = re.compile(r"^-?\d+$") NUMBER_PATTERN: re.Pattern[str] = re.compile(r"^-?\d+(\.\d+)?$") TRUTHY_PATTERN: re.Pattern[str] = re.compile(r"^(y|yes|t|true|on|1)$", re.IGNORECASE) FALSY_PATTERN: re.Pattern[str] = re.compile(r"^(n|no|f|false|off|0)$", re.IGNORECASE) +def _is_valid_absolute_path_string(value: str) -> bool: + """Return whether *value* is a portable absolute-path representation. + + Path recognition is intentionally a small sequence of bounded operations, + rather than a nested regular expression. This helper accepts POSIX, + UNC-style, and drive-qualified Windows absolute paths without depending on + the operating system running the library. It rejects characters that are + invalid in Windows path components and control characters that should not + reach a filesystem boundary. + """ + starts_with_separator = value.startswith(("/", "\\")) + starts_with_drive = ( + len(value) >= 3 + and value[0] in "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + and value[1] == ":" + and value[2] in {"/", "\\"} + ) + if not (starts_with_separator or starts_with_drive): + return False + + return not any( + character in '<>:"|?*\n\r\x00' and not (character == ":" and index == 1 and starts_with_drive) + for index, character in enumerate(value) + ) + + class ConversionError(ValueError): """Custom error class for handling conversion failures. @@ -222,9 +246,9 @@ def string_to_path(val: str | bytes | os.PathLike[str] | None, raise_on_error: b if raise_on_error: raise ConversionError(Path, val) from exc return None - # Ensure val is converted to string before matching + # Normalize before applying portable, linear-time path validation. val = str(val) - if not PATH_PATTERN.match(val): + if not _is_valid_absolute_path_string(val): raise ConversionError(Path, val) return Path(val) except (ValueError, TypeError) as exc: @@ -435,8 +459,8 @@ def reconstruct_special_type(converted_obj: str, fail_silently: bool = False) -> return string_to_date(converted_obj) if TIME_PATTERN.match(converted_obj): return string_to_time(converted_obj) - if PATH_PATTERN.match(converted_obj): - return pathlib.Path(converted_obj) + if _is_valid_absolute_path_string(converted_obj): + return Path(converted_obj) if TRUTHY_PATTERN.match(converted_obj) or FALSY_PATTERN.match(converted_obj): return string_to_bool(converted_obj) if NUMBER_PATTERN.match(converted_obj): diff --git a/packages/extended-data/tests/core/test_type_utils.py b/packages/extended-data/tests/core/test_type_utils.py index 7a21149..2443c46 100644 --- a/packages/extended-data/tests/core/test_type_utils.py +++ b/packages/extended-data/tests/core/test_type_utils.py @@ -717,6 +717,27 @@ def test_reconstruct_special_type_fail_silently() -> None: assert reconstruct_special_type("not a number", fail_silently=True) == "not a number" +@pytest.mark.parametrize( + "value", + [ + "/" + ("safe/" * 10_000) + "forbidden?", + "/path/with:colon", + "/path/with\nnewline", + "relative/path", + "C:relative\\path", + ], +) +def test_string_to_path_rejects_invalid_or_adversarial_path_text(value: str) -> None: + """Reject invalid paths without a backtracking regular-expression matcher.""" + assert string_to_path(value) is None + + +@pytest.mark.parametrize("value", ["/srv/config.yaml", r"C:\\config\\app.yaml", "D:/config/app.yaml"]) +def test_path_reconstruction_supports_portable_absolute_paths(value: str) -> None: + """Recognize supported absolute path forms when reconstructing values.""" + assert reconstruct_special_type(value) == Path(value) + + @pytest.mark.parametrize( ("obj", "expected"), [ diff --git a/packages/extended-data/tests/examples/test_safe_examples.py b/packages/extended-data/tests/examples/test_safe_examples.py index 92ea684..cc98a41 100644 --- a/packages/extended-data/tests/examples/test_safe_examples.py +++ b/packages/extended-data/tests/examples/test_safe_examples.py @@ -64,6 +64,13 @@ def _readme_usage_snippet() -> str: return match.group("code") +def _markdown_python_block(path: Path, block_number: int) -> str: + """Return a one-based Python fence from a public Markdown document.""" + blocks = [match.group("code") for match in PYTHON_MARKDOWN_BLOCK_RE.finditer(path.read_text(encoding="utf-8"))] + assert len(blocks) >= block_number, f"{path} is missing Python block {block_number}" + return blocks[block_number - 1] + + def _rst_python_code_blocks(text: str) -> list[str]: blocks: list[str] = [] lines = text.splitlines() @@ -155,6 +162,34 @@ def test_readme_usage_snippet_runs(tmp_path: Path) -> None: assert result.returncode == 0, f"README usage snippet failed\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" +@pytest.mark.parametrize( + ("markdown_path", "block_number"), + [ + (WORKSPACE_ROOT / "README.md", 1), + (WORKSPACE_ROOT / "docs" / "guides" / "getting-started.md", 1), + (WORKSPACE_ROOT / "docs" / "guides" / "agentic-consumers.md", 1), + ], +) +def test_public_quickstart_snippets_run(markdown_path: Path, block_number: int, tmp_path: Path) -> None: + """Keep the primary human and agent-facing quickstarts executable.""" + env = os.environ.copy() + env.pop("OVERRIDE_STDIN", None) + + result = subprocess.run( + [sys.executable, "-c", _markdown_python_block(markdown_path, block_number)], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + + assert result.returncode == 0, ( + f"{markdown_path.name} Python block {block_number} failed\nSTDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + + def test_documentation_python_snippets_compile() -> None: """Documentation snippets may be conceptual, but they should remain valid Python.""" markdown_paths = [REPO_ROOT / "README.md"] diff --git a/packages/pytest-extended-data/README.md b/packages/pytest-extended-data/README.md index afdbce2..ef1eab3 100644 --- a/packages/pytest-extended-data/README.md +++ b/packages/pytest-extended-data/README.md @@ -16,3 +16,22 @@ The plugin is exposed through the standard `pytest11` entry point. It provides: - `extended_data_value`: the payload wrapped as an `ExtendedData` value. - `assert_extended_shape(value, shape)`: assertion helper for shape checks. - `assert_builtin_round_trip(value, expected)`: assertion helper for export-boundary checks. + +Use the fixtures as consumer-facing contract checks, rather than duplicating +private container implementation details in every downstream package: + +```python +def test_configuration_boundary(extended_data_value): + assert extended_data_value["service"]["name"] == "api" + assert extended_data_value.as_builtin() == { + "service": {"name": "api", "ports": [8080, 8443]}, + "enabled": True, + } +``` + +The runtime package deliberately has no pytest plugin. Install this package +only in test environments so production dependencies remain focused on data +handling. + +See the [Package Surface guide](https://extended-data.dev/guides/package-surface.html) +for the runtime and plugin split. diff --git a/uv.lock b/uv.lock index b518f06..68cc6ba 100644 --- a/uv.lock +++ b/uv.lock @@ -372,7 +372,7 @@ wheels = [ [[package]] name = "extended-data" -version = "8.5.0" +version = "8.5.1" source = { editable = "packages/extended-data" } dependencies = [ { name = "deepmerge" }, @@ -427,7 +427,7 @@ requires-dist = [ { name = "coverage", extras = ["toml"], marker = "extra == 'tests'", specifier = ">=7.6.0" }, { name = "deepmerge", specifier = ">=2.0" }, { name = "extended-data", extras = ["tests", "typing"], marker = "extra == 'dev'", editable = "packages/extended-data" }, - { name = "gitpython", specifier = ">=3.1.54" }, + { name = "gitpython", specifier = ">=3.1.58" }, { name = "hypothesis", marker = "extra == 'tests'", specifier = ">=6.100.2" }, { name = "inflection", specifier = ">=0.5.1" }, { name = "mypy", marker = "extra == 'typing'", specifier = ">=1.20.1" }, @@ -506,14 +506,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.54" +version = "3.1.59" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5e/d5/3da0b92033887033f4c27f2dd109a303c4ca62813c7b3bb2511edb4777de/gitpython-3.1.54.tar.gz", hash = "sha256:53f2085e24a2cda300eed7c3fc5f1559ae289634b725e98acaf4791940247aa0", size = 225076, upload-time = "2026-07-22T04:08:51.403Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/dc/126b28e76b24a9268ba931ad3e012f71ebdadf62fd9f17758f7074bb0b20/gitpython-3.1.59.tar.gz", hash = "sha256:0a1475cfdc38a5bfba1a3e9a4a9da52a39749ecec322b772915c019f94e5b7e4", size = 230445, upload-time = "2026-08-10T12:03:20.271Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/b9/876f442a28df5c068ca69b0122d5c35e65fd2d2fa9992ea5cb5944ea00a6/gitpython-3.1.54-py3-none-any.whl", hash = "sha256:b90d7b3d9bc0238681d24369130826f0dcdb0ceaa45db67cf1d4ffa4c302dedf", size = 216575, upload-time = "2026-07-22T04:08:50.05Z" }, + { url = "https://files.pythonhosted.org/packages/ef/ed/ae57eb7d344f43f87b74b3a281ead6ec7d6394eef72a7b1dcb28dd089550/gitpython-3.1.59-py3-none-any.whl", hash = "sha256:67a82f537384578643624c8b2c531938a9b82be431663e575dcf638526631d4c", size = 220996, upload-time = "2026-08-10T12:03:18.804Z" }, ] [[package]]