From 5658dfa0234f565bbe886c452a29ba8c192772af Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:43:42 +0000 Subject: [PATCH 1/7] Remove unverified legacy endpoint lists --- .gitignore | 12 ++++++++++ 5.txt | 39 --------------------------------- 6.txt | 36 ------------------------------ README.md | 37 ++++++++++++++++++++++++++++++- SECURITY.md | 24 ++++++++++++++++++++ checker_2022-06-04_10.33.29.txt | 15 ------------- 6 files changed, 72 insertions(+), 91 deletions(-) create mode 100644 .gitignore delete mode 100644 5.txt delete mode 100644 6.txt create mode 100644 SECURITY.md delete mode 100644 checker_2022-06-04_10.33.29.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2ac43c9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.venv/ +dist/ +build/ + +# Raw endpoint corpora may contain third-party infrastructure and stay local. +*.endpoints +data/raw/ diff --git a/5.txt b/5.txt deleted file mode 100644 index d91ea28..0000000 --- a/5.txt +++ /dev/null @@ -1,39 +0,0 @@ -62.113.115.94:16072 -72.195.114.169:4145 -72.195.34.58:4145 -72.49.49.11:31034 -5.161.93.53:1080 -192.252.214.20:15864 -208.102.51.6:58208 -70.166.167.55:57745 -72.221.196.157:35904 -67.201.33.10:25283 -72.221.172.203:4145 -218.64.255.198:7302 -193.252.50.1:3820 -67.201.33.9:25280 -150.136.189.225:80 -192.111.130.2:4145 -62.113.115.94:16072 -72.206.181.97:64943 -72.195.114.169:4145 -72.195.34.58:4145 -72.221.232.155:4145 -72.49.49.11:31034 -5.161.93.53:1080 -72.221.164.34:60671 -192.252.214.20:15864 -208.102.51.6:58208 -72.206.181.103:4145 -72.217.216.239:4145 -70.166.167.55:57745 -72.221.196.157:35904 -70.166.167.38:57728 -72.206.181.105:64935 -67.201.33.10:25283 -72.221.172.203:4145 -103.156.141.237:1080 -66.29.154.105:1080 -218.64.255.198:7302 -193.252.50.1:3820 -67.201.33.9:25280 \ No newline at end of file diff --git a/6.txt b/6.txt deleted file mode 100644 index d835d5f..0000000 --- a/6.txt +++ /dev/null @@ -1,36 +0,0 @@ -72.49.49.11:31034 -69.61.200.104:36181 -66.42.224.229:41679 -192.252.208.70:14282 -67.201.33.10:25283 -192.111.130.2:4145 -192.252.209.155:14455 -192.111.137.34:18765 -192.111.139.165:4145 -192.111.139.162:4145 -72.195.34.60:27391 -24.249.199.12:4145 -72.210.252.137:4145 -70.166.167.38:57728 -72.206.181.105:64935 -72.195.34.35:27360 -72.206.181.123:4145 -72.221.196.157:35904 -72.217.216.239:4145 -72.206.181.103:4145 -72.221.232.152:4145 -72.210.208.101:4145 -24.249.199.4:4145 -72.221.172.203:4145 -72.195.34.58:4145 -72.221.232.155:4145 -72.206.181.97:64943 -198.8.94.170:4145 -185.26.144.197:35581 -151.80.252.69:64246 -188.165.254.122:9420 -51.77.141.29:1080 -146.56.173.56:22738 -151.80.252.69:7532 -45.10.71.15:8888 -192.111.135.17:18302 \ No newline at end of file diff --git a/README.md b/README.md index 94d2833..fb88021 100644 --- a/README.md +++ b/README.md @@ -1 +1,36 @@ -# Hello-World \ No newline at end of file +# Netveil + +Netveil is being rebuilt as an offline, privacy-preserving audit tool for +network-endpoint corpora. + +## Current status + +This repository is at **rehabilitation phase 0**. Its former working tree +contained unverified third-party IP address and port lists collected in 2022. +Those live endpoint lists have been removed from the current tree: they were +not suitable fixtures, did not include provenance or consent, and must not be +interpreted as working services. + +The old values remain in Git history until a separately coordinated history +rewrite. Do not use them for connection attempts, availability checks, or +security testing. + +## Intended direction + +The next incremental releases will add: + +- a strict offline parser and canonical endpoint representation; +- deterministic duplicate, range, and schema-quality checks; +- irreversible, domain-separated anonymisation for reportable aggregates; +- content-addressed audit receipts and reproducible CLI evidence; +- tests proving that the default workflow never opens a network socket. + +No scanner, proxy checker, or network probe is currently shipped. + +## Safety boundary + +Netveil will operate on local files only and fail closed on malformed input. +Only synthetic or explicitly redistributable fixtures will be committed. +Testing systems you do not own or lack permission to assess is out of scope. + +See [SECURITY.md](SECURITY.md) for responsible-use and disclosure guidance. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..0166955 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,24 @@ +# Security policy + +## Supported state + +Netveil is in rehabilitation phase 0 and has no released executable yet. +Security fixes target the default branch and the latest open rehabilitation +pull request. + +## Data handling + +Do not submit real credentials, access tokens, private network addresses, or +third-party endpoint inventories in an issue or pull request. Use the future +synthetic fixtures when demonstrating a parser or report. + +The historical repository contains unverified public endpoint strings. Their +presence does not grant permission to connect to, probe, or test those systems. +The current tree intentionally contains none of those values. + +## Reporting a vulnerability + +Use GitHub's private vulnerability reporting feature for this repository when +available. Include the affected commit, a minimal reproduction with synthetic +data, the expected safety boundary, and the observed behaviour. Do not include +secrets or personal data. diff --git a/checker_2022-06-04_10.33.29.txt b/checker_2022-06-04_10.33.29.txt deleted file mode 100644 index 35e7bfe..0000000 --- a/checker_2022-06-04_10.33.29.txt +++ /dev/null @@ -1,15 +0,0 @@ -62.113.115.94:16072 -72.195.114.169:4145 -72.195.34.58:4145 -72.49.49.11:31034 -5.161.93.53:1080 -192.252.214.20:15864 -208.102.51.6:58208 -70.166.167.55:57745 -72.221.196.157:35904 -67.201.33.10:25283 -72.221.172.203:4145 -218.64.255.198:7302 -193.252.50.1:3820 -67.201.33.9:25280 -150.136.189.225:80 \ No newline at end of file From f0c6e4d5e84bb34fe01e1c28d14010a2601124d1 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:15:25 +0000 Subject: [PATCH 2/7] Build fail-closed offline endpoint parser --- .gitignore | 2 + LICENSE | 21 ++++ README.md | 62 ++++++++-- SECURITY.md | 18 ++- pyproject.toml | 39 ++++++ src/netveil/__init__.py | 27 +++++ src/netveil/model.py | 188 ++++++++++++++++++++++++++++ src/netveil/parser.py | 159 ++++++++++++++++++++++++ src/netveil/py.typed | 1 + tests/test_model.py | 133 ++++++++++++++++++++ tests/test_packaging.py | 15 +++ tests/test_parser.py | 263 ++++++++++++++++++++++++++++++++++++++++ 12 files changed, 913 insertions(+), 15 deletions(-) create mode 100644 LICENSE create mode 100644 pyproject.toml create mode 100644 src/netveil/__init__.py create mode 100644 src/netveil/model.py create mode 100644 src/netveil/parser.py create mode 100644 src/netveil/py.typed create mode 100644 tests/test_model.py create mode 100644 tests/test_packaging.py create mode 100644 tests/test_parser.py diff --git a/.gitignore b/.gitignore index 2ac43c9..98729d4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ __pycache__/ .ruff_cache/ .mypy_cache/ .venv/ +.coverage +*.egg-info/ dist/ build/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a9918bd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Omar Ibrahim + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index fb88021..6e12aed 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ # Netveil -Netveil is being rebuilt as an offline, privacy-preserving audit tool for -network-endpoint corpora. +Netveil is being rebuilt as an offline, fail-closed audit tool for +network-endpoint corpora with an explicit privacy boundary. ## Current status -This repository is at **rehabilitation phase 0**. Its former working tree +This repository is at **rehabilitation phase 1**. Its former working tree contained unverified third-party IP address and port lists collected in 2022. Those live endpoint lists have been removed from the current tree: they were not suitable fixtures, did not include provenance or consent, and must not be @@ -15,22 +15,66 @@ The old values remain in Git history until a separately coordinated history rewrite. Do not use them for connection attempts, availability checks, or security testing. +The first shipped layer is a dependency-free, offline parser with: + +- strict UTF-8, line-ending, IP-address, and port validation; +- canonical IPv4 and bracketed IPv6 representations; +- deterministic duplicate detection across equivalent IPv6 spellings; +- explicit network-scope classification, including documentation ranges; +- content binding to the SHA-256 digest of the exact input bytes; +- bounded input and safe errors that never repeat endpoint values. + +It does not import a networking client, resolve DNS, or open sockets. + +### Phase-1 privacy boundary + +Phase 1 keeps processing local and returns redacted parse failures, but parsed +`Endpoint` and `EndpointCorpus` objects intentionally retain raw canonical IP +addresses. Treat those objects as sensitive: do not log, serialize, publish, or +attach them to issues when they came from a real corpus. + +Anonymisation is **not** implemented yet. Irreversible, domain-separated +identifiers for aggregate reports remain a future phase and are listed below. +The current code therefore claims locality and redacted errors, not anonymous +output. + +## Quick check + +```bash +python3 -m venv .venv +.venv/bin/python -m pip install -e . +.venv/bin/python -m unittest discover -s tests -v +``` + +The parser is a library boundary for now: + +```python +from netveil import parse_corpus + +corpus = parse_corpus(b"192.0.2.10:443\n[2001:db8::10]:8443\n") +print(corpus.unique_count) +print(corpus.source_sha256) +``` + +The addresses above are IETF documentation ranges, not live fixtures. + ## Intended direction The next incremental releases will add: -- a strict offline parser and canonical endpoint representation; -- deterministic duplicate, range, and schema-quality checks; +- aggregate duplicate, range, and schema-quality reports; - irreversible, domain-separated anonymisation for reportable aggregates; - content-addressed audit receipts and reproducible CLI evidence; -- tests proving that the default workflow never opens a network socket. +- installed-artifact guards preserving the no-network boundary as reporting + layers are added. -No scanner, proxy checker, or network probe is currently shipped. +No scanner, proxy checker, or network probe is shipped. ## Safety boundary -Netveil will operate on local files only and fail closed on malformed input. -Only synthetic or explicitly redistributable fixtures will be committed. +Netveil operates on caller-supplied local bytes only and fails closed on +malformed input. Only synthetic or explicitly redistributable fixtures will be +committed. Testing systems you do not own or lack permission to assess is out of scope. See [SECURITY.md](SECURITY.md) for responsible-use and disclosure guidance. diff --git a/SECURITY.md b/SECURITY.md index 0166955..e18e8d8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,15 +2,21 @@ ## Supported state -Netveil is in rehabilitation phase 0 and has no released executable yet. -Security fixes target the default branch and the latest open rehabilitation -pull request. +Netveil is in rehabilitation phase 1. It ships an offline parser library but +no command-line scanner, network probe, or stable release. Security fixes +target the default branch and the latest open rehabilitation pull request. ## Data handling -Do not submit real credentials, access tokens, private network addresses, or -third-party endpoint inventories in an issue or pull request. Use the future -synthetic fixtures when demonstrating a parser or report. +Do not submit real credentials, access tokens, private-network inventories, or +third-party endpoint inventories in an issue or pull request. Use IETF +documentation ranges or clearly synthetic special-use addresses when +demonstrating the parser. + +Phase-1 `Endpoint` and `EndpointCorpus` objects retain raw canonical addresses. +They are not anonymized and must not be logged, published, or attached to an +issue when created from a real corpus. Redacted exceptions protect error +messages only; irreversible report identifiers are a future phase. The historical repository contains unverified public endpoint strings. Their presence does not grant permission to connect to, probe, or test those systems. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..87ff32f --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,39 @@ +[build-system] +requires = ["setuptools==83.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "netveil-audit" +version = "0.1.0" +description = "Offline fail-closed parser for network-endpoint corpora" +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +license-files = ["LICENSE"] +authors = [{name = "Omar Ibrahim"}] +dependencies = [] + +[tool.setuptools] +package-dir = {"" = "src"} +packages = ["netveil"] + +[tool.setuptools.package-data] +netveil = ["py.typed"] + +[tool.ruff] +line-length = 88 +target-version = "py311" + +[tool.mypy] +python_version = "3.11" +strict = true +files = ["src", "tests"] + +[tool.coverage.run] +branch = true +source = ["netveil"] + +[tool.coverage.report] +fail_under = 99 +precision = 2 +show_missing = true diff --git a/src/netveil/__init__.py b/src/netveil/__init__.py new file mode 100644 index 0000000..c9c833c --- /dev/null +++ b/src/netveil/__init__.py @@ -0,0 +1,27 @@ +"""Offline endpoint parsing with redacted failures and no network access.""" + +from netveil.model import ( + Endpoint, + EndpointCorpus, + EndpointScope, + IPVersion, +) +from netveil.parser import ( + MAX_INPUT_BYTES, + MAX_PHYSICAL_LINES, + EndpointParseError, + EndpointParseErrorCode, + parse_corpus, +) + +__all__ = [ + "MAX_INPUT_BYTES", + "MAX_PHYSICAL_LINES", + "Endpoint", + "EndpointCorpus", + "EndpointParseError", + "EndpointParseErrorCode", + "EndpointScope", + "IPVersion", + "parse_corpus", +] diff --git a/src/netveil/model.py b/src/netveil/model.py new file mode 100644 index 0000000..0c33914 --- /dev/null +++ b/src/netveil/model.py @@ -0,0 +1,188 @@ +"""Immutable endpoint corpus model with internal validated construction.""" + +from __future__ import annotations + +import hashlib +import ipaddress +from dataclasses import dataclass +from enum import Enum +from typing import Self, final + +_DOCUMENTATION_NETWORKS = ( + ipaddress.ip_network("192.0.2.0/24"), + ipaddress.ip_network("198.51.100.0/24"), + ipaddress.ip_network("203.0.113.0/24"), + ipaddress.ip_network("2001:db8::/32"), +) +_SHARED_ADDRESS_SPACE = ipaddress.ip_network("100.64.0.0/10") + + +class IPVersion(Enum): + """Supported IP address families.""" + + IPV4 = 4 + IPV6 = 6 + + +class EndpointScope(Enum): + """Privacy-relevant address classification.""" + + DOCUMENTATION = "documentation" + GLOBAL = "global" + PRIVATE = "private" + LOOPBACK = "loopback" + LINK_LOCAL = "link_local" + MULTICAST = "multicast" + SHARED = "shared" + SITE_LOCAL = "site_local" + UNSPECIFIED = "unspecified" + RESERVED = "reserved" + + +@final +@dataclass(frozen=True, slots=True, init=False) +class Endpoint: + """One canonical raw IP endpoint produced only by ``parse_corpus``.""" + + address: str + port: int + version: IPVersion + scope: EndpointScope + + def __new__(cls) -> Self: + """Block direct construction outside the validated parser boundary.""" + + raise TypeError("Endpoint objects are created by parse_corpus") + + @property + def canonical(self) -> str: + """Return the unambiguous canonical endpoint representation.""" + + if self.version is IPVersion.IPV6: + return f"[{self.address}]:{self.port}" + return f"{self.address}:{self.port}" + + +@final +@dataclass(frozen=True, slots=True, init=False) +class EndpointCorpus: + """A parsed corpus bound to exact bytes by the internal factory.""" + + source_sha256: str + source_bytes: int + physical_line_count: int + endpoints: tuple[Endpoint, ...] + unique_endpoints: tuple[Endpoint, ...] + + def __new__(cls) -> Self: + """Block direct construction outside the validated parser boundary.""" + + raise TypeError("EndpointCorpus objects are created by parse_corpus") + + @property + def endpoint_count(self) -> int: + return len(self.endpoints) + + @property + def unique_count(self) -> int: + return len(self.unique_endpoints) + + @property + def duplicate_count(self) -> int: + return self.endpoint_count - self.unique_count + + +def _scope( + address: ipaddress.IPv4Address | ipaddress.IPv6Address, +) -> EndpointScope: + """Classify one exact stdlib address without any network operation.""" + + if any(address in network for network in _DOCUMENTATION_NETWORKS): + return EndpointScope.DOCUMENTATION + if address.is_unspecified: + return EndpointScope.UNSPECIFIED + if address.is_loopback: + return EndpointScope.LOOPBACK + if address.is_multicast: + return EndpointScope.MULTICAST + if address.is_link_local: + return EndpointScope.LINK_LOCAL + if isinstance(address, ipaddress.IPv6Address) and address.is_site_local: + return EndpointScope.SITE_LOCAL + if address in _SHARED_ADDRESS_SPACE: + return EndpointScope.SHARED + if address.is_reserved: + return EndpointScope.RESERVED + if address.is_private: + return EndpointScope.PRIVATE + return EndpointScope.GLOBAL + + +def _create_endpoint( + address: ipaddress.IPv4Address | ipaddress.IPv6Address, + *, + port: int, +) -> Endpoint: + """Create an endpoint after checking every stored field invariant.""" + + if type(address) not in (ipaddress.IPv4Address, ipaddress.IPv6Address): + raise TypeError("address must be an exact IPv4Address or IPv6Address") + if type(port) is not int: + raise TypeError("port must be an exact int") + if not 1 <= port <= 65_535: + raise ValueError("port must be in the range 1..65535") + if isinstance(address, ipaddress.IPv6Address) and address.scope_id is not None: + raise ValueError("scoped IPv6 addresses are not supported") + + endpoint: Endpoint = object.__new__(Endpoint) + object.__setattr__(endpoint, "address", address.compressed) + object.__setattr__(endpoint, "port", port) + object.__setattr__( + endpoint, + "version", + IPVersion.IPV6 + if isinstance(address, ipaddress.IPv6Address) + else IPVersion.IPV4, + ) + object.__setattr__(endpoint, "scope", _scope(address)) + return endpoint + + +def _create_corpus( + *, + payload: bytes, + physical_line_count: int, + endpoints: tuple[Endpoint, ...], +) -> EndpointCorpus: + """Create a corpus while deriving all identity and uniqueness fields.""" + + if type(payload) is not bytes: + raise TypeError("payload must be exact bytes") + if type(physical_line_count) is not int: + raise TypeError("physical_line_count must be an exact int") + if type(endpoints) is not tuple: + raise TypeError("endpoints must be an exact tuple") + if physical_line_count < 0: + raise ValueError("physical_line_count must be non-negative") + if not endpoints: + raise ValueError("a corpus must contain at least one endpoint") + if physical_line_count < len(endpoints): + raise ValueError("physical_line_count cannot be smaller than endpoint_count") + if any(type(endpoint) is not Endpoint for endpoint in endpoints): + raise TypeError("every endpoint must come from the validated factory") + + unique_by_canonical: dict[str, Endpoint] = {} + for endpoint in endpoints: + unique_by_canonical.setdefault(endpoint.canonical, endpoint) + + corpus: EndpointCorpus = object.__new__(EndpointCorpus) + object.__setattr__(corpus, "source_sha256", hashlib.sha256(payload).hexdigest()) + object.__setattr__(corpus, "source_bytes", len(payload)) + object.__setattr__(corpus, "physical_line_count", physical_line_count) + object.__setattr__(corpus, "endpoints", endpoints) + object.__setattr__( + corpus, + "unique_endpoints", + tuple(unique_by_canonical.values()), + ) + return corpus diff --git a/src/netveil/parser.py b/src/netveil/parser.py new file mode 100644 index 0000000..88d333e --- /dev/null +++ b/src/netveil/parser.py @@ -0,0 +1,159 @@ +"""Fail-closed parser for local IP endpoint corpora.""" + +from __future__ import annotations + +import ipaddress +import re +from enum import Enum +from typing import NoReturn + +from netveil.model import ( + Endpoint, + EndpointCorpus, + _create_corpus, + _create_endpoint, +) + +MAX_INPUT_BYTES = 8 * 1024 * 1024 +MAX_PHYSICAL_LINES = 100_000 + +_IPV4_ENDPOINT = re.compile(r"^(?P
[^:[\]]+):(?P[0-9]+)$") +_IPV6_ENDPOINT = re.compile(r"^\[(?P
[^\[\]]+)\]:(?P[0-9]+)$") +_NON_LF_LINE_SEPARATORS = frozenset( + { + "\v", + "\f", + "\x1c", + "\x1d", + "\x1e", + "\x85", + "\u2028", + "\u2029", + } +) + + +class EndpointParseErrorCode(Enum): + """Stable public error codes that do not expose source values.""" + + EMPTY_CORPUS = "empty_corpus" + INPUT_TOO_LARGE = "input_too_large" + INVALID_ADDRESS = "invalid_address" + INVALID_LINE_ENDING = "invalid_line_ending" + INVALID_PORT = "invalid_port" + INVALID_SYNTAX = "invalid_syntax" + INVALID_UTF8 = "invalid_utf8" + NON_CANONICAL_PORT = "non_canonical_port" + TOO_MANY_LINES = "too_many_lines" + + +class EndpointParseError(ValueError): + """A redacted parse failure.""" + + def __init__( + self, + code: EndpointParseErrorCode, + *, + line_number: int | None = None, + ) -> None: + self.code = code + self.line_number = line_number + location = "" if line_number is None else f" at line {line_number}" + super().__init__(f"endpoint corpus rejected: {code.value}{location}") + + +def _fail( + code: EndpointParseErrorCode, + *, + line_number: int | None = None, +) -> NoReturn: + raise EndpointParseError(code, line_number=line_number) + + +def _parse_port(raw_port: str, *, line_number: int) -> int: + if len(raw_port) > 1 and raw_port.startswith("0"): + _fail(EndpointParseErrorCode.NON_CANONICAL_PORT, line_number=line_number) + if len(raw_port) > 5: + _fail(EndpointParseErrorCode.INVALID_PORT, line_number=line_number) + port = int(raw_port) + if not 1 <= port <= 65_535: + _fail(EndpointParseErrorCode.INVALID_PORT, line_number=line_number) + return port + + +def _parse_endpoint(raw_line: str, *, line_number: int) -> Endpoint: + if raw_line != raw_line.strip() or any( + character.isspace() for character in raw_line + ): + _fail(EndpointParseErrorCode.INVALID_SYNTAX, line_number=line_number) + + ipv6_match = _IPV6_ENDPOINT.fullmatch(raw_line) + ipv4_match = _IPV4_ENDPOINT.fullmatch(raw_line) + if ipv6_match is not None: + match = ipv6_match + elif ipv4_match is not None: + match = ipv4_match + else: + _fail(EndpointParseErrorCode.INVALID_SYNTAX, line_number=line_number) + + raw_address = match.group("address") + raw_port = match.group("port") + if "%" in raw_address: + _fail(EndpointParseErrorCode.INVALID_ADDRESS, line_number=line_number) + try: + parsed_address = ipaddress.ip_address(raw_address) + except ValueError: + _fail(EndpointParseErrorCode.INVALID_ADDRESS, line_number=line_number) + + if (ipv6_match is not None) != isinstance( + parsed_address, + ipaddress.IPv6Address, + ): + _fail(EndpointParseErrorCode.INVALID_SYNTAX, line_number=line_number) + + return _create_endpoint( + parsed_address, + port=_parse_port(raw_port, line_number=line_number), + ) + + +def parse_corpus(payload: bytes) -> EndpointCorpus: + """Parse exact local bytes without resolving or contacting any endpoint.""" + + if type(payload) is not bytes: + raise TypeError("payload must be exact bytes") + if len(payload) > MAX_INPUT_BYTES: + _fail(EndpointParseErrorCode.INPUT_TOO_LARGE) + if b"\r" in payload.replace(b"\r\n", b""): + _fail(EndpointParseErrorCode.INVALID_LINE_ENDING) + try: + text = payload.decode("utf-8", errors="strict") + except UnicodeDecodeError: + _fail(EndpointParseErrorCode.INVALID_UTF8) + if any(character in _NON_LF_LINE_SEPARATORS for character in text): + _fail(EndpointParseErrorCode.INVALID_LINE_ENDING) + + normalized_text = text.replace("\r\n", "\n") + if normalized_text: + physical_lines = normalized_text.split("\n") + if physical_lines[-1] == "": + physical_lines.pop() + else: + physical_lines = [] + if len(physical_lines) > MAX_PHYSICAL_LINES: + _fail(EndpointParseErrorCode.TOO_MANY_LINES) + + endpoints: list[Endpoint] = [] + for line_number, raw_line in enumerate(physical_lines, start=1): + if not raw_line or raw_line.startswith("#"): + continue + endpoints.append(_parse_endpoint(raw_line, line_number=line_number)) + + if not endpoints: + _fail(EndpointParseErrorCode.EMPTY_CORPUS) + + return _create_corpus( + payload=payload, + physical_line_count=len(physical_lines), + endpoints=tuple(endpoints), + ) diff --git a/src/netveil/py.typed b/src/netveil/py.typed new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/netveil/py.typed @@ -0,0 +1 @@ + diff --git a/tests/test_model.py b/tests/test_model.py new file mode 100644 index 0000000..3314f68 --- /dev/null +++ b/tests/test_model.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import hashlib +import ipaddress +import unittest +from typing import cast + +from netveil import Endpoint, EndpointCorpus, EndpointScope, IPVersion +from netveil.model import _create_corpus, _create_endpoint + + +class ModelConstructionBoundaryTests(unittest.TestCase): + def test_public_constructors_are_blocked(self) -> None: + with self.assertRaisesRegex( + TypeError, + "^Endpoint objects are created by parse_corpus$", + ): + Endpoint() + with self.assertRaisesRegex( + TypeError, + "^EndpointCorpus objects are created by parse_corpus$", + ): + EndpointCorpus() + + with self.assertRaises(TypeError): + Endpoint( # type: ignore[call-arg] + "not-an-ip", + 0, + IPVersion.IPV4, + EndpointScope.DOCUMENTATION, + ) + with self.assertRaises(TypeError): + EndpointCorpus( # type: ignore[call-arg] + "not-a-sha256", + -1, + -1, + (), + (), + ) + + def test_endpoint_factory_enforces_exact_types_and_domains(self) -> None: + address = ipaddress.IPv4Address("192.0.2.1") + + with self.assertRaisesRegex(TypeError, "^address must be an exact"): + _create_endpoint("192.0.2.1", port=443) # type: ignore[arg-type] + with self.assertRaisesRegex(TypeError, "^port must be an exact int$"): + _create_endpoint(address, port=True) + for port in (0, 65_536): + with ( + self.subTest(port=port), + self.assertRaisesRegex(ValueError, "^port must be in"), + ): + _create_endpoint(address, port=port) + with self.assertRaisesRegex(ValueError, "^scoped IPv6"): + _create_endpoint( + ipaddress.IPv6Address("fe80::1%eth0"), + port=443, + ) + + def test_corpus_factory_derives_identity_counts_and_uniqueness(self) -> None: + payload = b"192.0.2.1:443\n192.0.2.1:443\n" + endpoint = _create_endpoint( + ipaddress.IPv4Address("192.0.2.1"), + port=443, + ) + corpus = _create_corpus( + payload=payload, + physical_line_count=2, + endpoints=(endpoint, endpoint), + ) + + self.assertEqual(corpus.source_sha256, hashlib.sha256(payload).hexdigest()) + self.assertEqual(corpus.source_bytes, len(payload)) + self.assertEqual(corpus.physical_line_count, 2) + self.assertEqual(corpus.endpoints, (endpoint, endpoint)) + self.assertEqual(corpus.unique_endpoints, (endpoint,)) + self.assertEqual(corpus.duplicate_count, 1) + + def test_corpus_factory_rejects_impossible_inputs(self) -> None: + endpoint = _create_endpoint( + ipaddress.IPv4Address("192.0.2.1"), + port=443, + ) + + with self.assertRaisesRegex(TypeError, "^payload must be exact bytes$"): + _create_corpus( + payload=cast(bytes, bytearray(b"x")), + physical_line_count=1, + endpoints=(endpoint,), + ) + with self.assertRaisesRegex( + TypeError, + "^physical_line_count must be an exact int$", + ): + _create_corpus( + payload=b"x", + physical_line_count=True, + endpoints=(endpoint,), + ) + with self.assertRaisesRegex(TypeError, "^endpoints must be an exact tuple$"): + _create_corpus( + payload=b"x", + physical_line_count=1, + endpoints=cast(tuple[Endpoint, ...], [endpoint]), + ) + with self.assertRaisesRegex(ValueError, "^physical_line_count must be"): + _create_corpus( + payload=b"x", + physical_line_count=-1, + endpoints=(endpoint,), + ) + with self.assertRaisesRegex(ValueError, "^a corpus must contain"): + _create_corpus( + payload=b"", + physical_line_count=0, + endpoints=(), + ) + with self.assertRaisesRegex(ValueError, "^physical_line_count cannot"): + _create_corpus( + payload=b"x", + physical_line_count=1, + endpoints=(endpoint, endpoint), + ) + with self.assertRaisesRegex(TypeError, "^every endpoint must come"): + _create_corpus( + payload=b"x", + physical_line_count=1, + endpoints=cast(tuple[Endpoint, ...], (object(),)), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..cbe0b83 --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,15 @@ +from __future__ import annotations + +import unittest +from importlib import resources + + +class PackagingContractTests(unittest.TestCase): + def test_pep561_marker_is_packaged(self) -> None: + marker = resources.files("netveil").joinpath("py.typed") + self.assertTrue(marker.is_file()) + self.assertIn(marker.read_bytes(), (b"", b"\n")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..0c04e04 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import hashlib +import ipaddress +import socket +import unittest +from unittest.mock import patch + +import netveil.model as model_module +from netveil import ( + MAX_INPUT_BYTES, + MAX_PHYSICAL_LINES, + EndpointParseError, + EndpointParseErrorCode, + EndpointScope, + IPVersion, + parse_corpus, +) + + +class SyntheticGlobalIPv4(ipaddress.IPv4Address): + """Exercise the global branch without committing a live endpoint.""" + + @property + def is_unspecified(self) -> bool: + return False + + @property + def is_loopback(self) -> bool: + return False + + @property + def is_multicast(self) -> bool: + return False + + @property + def is_link_local(self) -> bool: + return False + + @property + def is_reserved(self) -> bool: + return False + + @property + def is_private(self) -> bool: + return False + + +class DecodeOverridingBytes(bytes): + def decode( + self, + encoding: str = "utf-8", + errors: str = "strict", + ) -> str: + del encoding, errors + return "192.0.2.1:443\n" + + +class EndpointParserHappyPathTests(unittest.TestCase): + def test_parses_canonical_ipv4_and_ipv6_without_network_access(self) -> None: + payload = ( + b"# Synthetic IETF documentation ranges only\n" + b"192.0.2.10:443\n" + b"[2001:0DB8:0:0:0:0:0:10]:8443\n" + ) + with ( + patch.object(socket, "socket", side_effect=AssertionError("network")), + patch.object( + socket, + "getaddrinfo", + side_effect=AssertionError("resolution"), + ), + ): + corpus = parse_corpus(payload) + + self.assertEqual(corpus.source_sha256, hashlib.sha256(payload).hexdigest()) + self.assertEqual(corpus.source_bytes, len(payload)) + self.assertEqual(corpus.physical_line_count, 3) + self.assertEqual(corpus.endpoint_count, 2) + self.assertEqual(corpus.unique_count, 2) + self.assertEqual(corpus.duplicate_count, 0) + self.assertEqual(corpus.endpoints[0].canonical, "192.0.2.10:443") + self.assertEqual(corpus.endpoints[0].version, IPVersion.IPV4) + self.assertEqual(corpus.endpoints[0].scope, EndpointScope.DOCUMENTATION) + self.assertEqual(corpus.endpoints[1].canonical, "[2001:db8::10]:8443") + self.assertEqual(corpus.endpoints[1].version, IPVersion.IPV6) + self.assertEqual(corpus.endpoints[1].scope, EndpointScope.DOCUMENTATION) + + def test_accepts_crlf_and_preserves_exact_source_hash(self) -> None: + payload = b"198.51.100.8:80\r\n203.0.113.9:443\r\n" + corpus = parse_corpus(payload) + self.assertEqual(corpus.physical_line_count, 2) + self.assertEqual(corpus.source_sha256, hashlib.sha256(payload).hexdigest()) + + without_final_newline = parse_corpus(b"192.0.2.10:443") + self.assertEqual(without_final_newline.physical_line_count, 1) + + def test_equivalent_ipv6_spellings_are_duplicates(self) -> None: + corpus = parse_corpus(b"[2001:db8::1]:443\n[2001:0DB8:0:0:0:0:0:1]:443\n") + self.assertEqual(corpus.endpoint_count, 2) + self.assertEqual(corpus.unique_count, 1) + self.assertEqual(corpus.duplicate_count, 1) + self.assertEqual(corpus.unique_endpoints[0].canonical, "[2001:db8::1]:443") + + def test_classifies_privacy_relevant_scopes(self) -> None: + corpus = parse_corpus( + b"10.0.0.1:1\n" + b"127.0.0.1:2\n" + b"169.254.1.1:3\n" + b"224.0.0.1:4\n" + b"0.0.0.0:5\n" + b"100.64.0.1:6\n" + b"240.0.0.1:7\n" + b"[fec0::1]:8\n" + ) + self.assertEqual( + [endpoint.scope for endpoint in corpus.endpoints], + [ + EndpointScope.PRIVATE, + EndpointScope.LOOPBACK, + EndpointScope.LINK_LOCAL, + EndpointScope.MULTICAST, + EndpointScope.UNSPECIFIED, + EndpointScope.SHARED, + EndpointScope.RESERVED, + EndpointScope.SITE_LOCAL, + ], + ) + + def test_global_scope_branch_uses_no_live_endpoint_fixture(self) -> None: + address = SyntheticGlobalIPv4("192.0.2.1") + with patch.object(model_module, "_DOCUMENTATION_NETWORKS", ()): + self.assertEqual( + model_module._scope(address), + EndpointScope.GLOBAL, + ) + + +class EndpointParserFailureTests(unittest.TestCase): + def assert_rejected( + self, + payload: bytes, + code: EndpointParseErrorCode, + *, + line_number: int | None, + forbidden: str | None = None, + ) -> None: + with self.assertRaises(EndpointParseError) as raised: + parse_corpus(payload) + self.assertEqual(raised.exception.code, code) + self.assertEqual(raised.exception.line_number, line_number) + if forbidden is not None: + self.assertNotIn(forbidden, str(raised.exception)) + + def test_rejects_empty_or_comment_only_corpora(self) -> None: + for payload in (b"", b"\n", b"# no endpoints\n"): + with self.subTest(payload=payload): + self.assert_rejected( + payload, + EndpointParseErrorCode.EMPTY_CORPUS, + line_number=None, + ) + + def test_rejects_ambiguous_or_invalid_syntax(self) -> None: + cases = ( + (b" 192.0.2.1:443\n", "192.0.2.1"), + (b"192.0.2.1:443 # inline\n", "192.0.2.1"), + (b"2001:db8::1:443\n", "2001:db8"), + (b"[192.0.2.1]:443\n", "192.0.2.1"), + ) + for payload, forbidden in cases: + with self.subTest(payload=payload): + self.assert_rejected( + payload, + EndpointParseErrorCode.INVALID_SYNTAX, + line_number=1, + forbidden=forbidden, + ) + self.assert_rejected( + b"example.invalid:443\n", + EndpointParseErrorCode.INVALID_ADDRESS, + line_number=1, + forbidden="example.invalid", + ) + + def test_rejects_invalid_addresses_without_echoing_them(self) -> None: + for payload, forbidden in ( + (b"999.51.100.7:443\n", "999.51.100.7"), + (b"[fe80::1%eth0]:443\n", "eth0"), + ): + with self.subTest(payload=payload): + self.assert_rejected( + payload, + EndpointParseErrorCode.INVALID_ADDRESS, + line_number=1, + forbidden=forbidden, + ) + + def test_rejects_invalid_and_noncanonical_ports(self) -> None: + cases = ( + (b"192.0.2.1:0\n", EndpointParseErrorCode.INVALID_PORT), + (b"192.0.2.1:65536\n", EndpointParseErrorCode.INVALID_PORT), + (b"192.0.2.1:100000\n", EndpointParseErrorCode.INVALID_PORT), + (b"192.0.2.1:0443\n", EndpointParseErrorCode.NON_CANONICAL_PORT), + ) + for payload, code in cases: + with self.subTest(payload=payload): + self.assert_rejected(payload, code, line_number=1) + + def test_rejects_invalid_utf8_and_non_lf_line_endings(self) -> None: + self.assert_rejected( + b"\xff:443\n", + EndpointParseErrorCode.INVALID_UTF8, + line_number=None, + ) + self.assert_rejected( + b"192.0.2.1:443\r192.0.2.2:443\n", + EndpointParseErrorCode.INVALID_LINE_ENDING, + line_number=None, + ) + for separator in ( + "\u000b", + "\u000c", + "\u001c", + "\u001d", + "\u001e", + "\u0085", + "\u2028", + "\u2029", + ): + with self.subTest(separator=separator.encode().hex()): + self.assert_rejected( + f"192.0.2.1:443{separator}192.0.2.2:443\n".encode(), + EndpointParseErrorCode.INVALID_LINE_ENDING, + line_number=None, + ) + self.assert_rejected( + f"# comment{separator}hidden line\n192.0.2.1:443\n".encode(), + EndpointParseErrorCode.INVALID_LINE_ENDING, + line_number=None, + ) + + def test_enforces_resource_bounds(self) -> None: + self.assert_rejected( + b"x" * (MAX_INPUT_BYTES + 1), + EndpointParseErrorCode.INPUT_TOO_LARGE, + line_number=None, + ) + self.assert_rejected( + b"\n" * (MAX_PHYSICAL_LINES + 1), + EndpointParseErrorCode.TOO_MANY_LINES, + line_number=None, + ) + + def test_requires_exact_bytes_type(self) -> None: + with self.assertRaisesRegex(TypeError, "^payload must be exact bytes$"): + parse_corpus("192.0.2.1:443\n") # type: ignore[arg-type] + with self.assertRaisesRegex(TypeError, "^payload must be exact bytes$"): + parse_corpus(DecodeOverridingBytes(b"not an endpoint")) + + +if __name__ == "__main__": + unittest.main() From 2657fd08f94ee32ab850fca5ab08d22627ee4ea9 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:38:11 +0000 Subject: [PATCH 3/7] Add keyed aggregate privacy receipts --- MANIFEST.in | 2 + README.md | 93 ++++-- SECURITY.md | 41 ++- docs/privacy-protocol.md | 125 ++++++++ pyproject.toml | 4 +- src/netveil/__init__.py | 26 ++ src/netveil/privacy.py | 517 +++++++++++++++++++++++++++++++++ tests/test_privacy.py | 597 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 1378 insertions(+), 27 deletions(-) create mode 100644 MANIFEST.in create mode 100644 docs/privacy-protocol.md create mode 100644 src/netveil/privacy.py create mode 100644 tests/test_privacy.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..9c97a7a --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +include SECURITY.md +include docs/privacy-protocol.md diff --git a/README.md b/README.md index 6e12aed..9f9776d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ network-endpoint corpora with an explicit privacy boundary. ## Current status -This repository is at **rehabilitation phase 1**. Its former working tree +This repository is at **rehabilitation phase 2**. Its former working tree contained unverified third-party IP address and port lists collected in 2022. Those live endpoint lists have been removed from the current tree: they were not suitable fixtures, did not include provenance or consent, and must not be @@ -15,7 +15,7 @@ The old values remain in Git history until a separately coordinated history rewrite. Do not use them for connection attempts, availability checks, or security testing. -The first shipped layer is a dependency-free, offline parser with: +The first layer is a dependency-free, offline parser with: - strict UTF-8, line-ending, IP-address, and port validation; - canonical IPv4 and bracketed IPv6 representations; @@ -26,17 +26,43 @@ The first shipped layer is a dependency-free, offline parser with: It does not import a networking client, resolve DNS, or open sockets. -### Phase-1 privacy boundary - -Phase 1 keeps processing local and returns redacted parse failures, but parsed -`Endpoint` and `EndpointCorpus` objects intentionally retain raw canonical IP -addresses. Treat those objects as sensitive: do not log, serialize, publish, or -attach them to issues when they came from a real corpus. - -Anonymisation is **not** implemented yet. Irreversible, domain-separated -identifiers for aggregate reports remain a future phase and are listed below. -The current code therefore claims locality and redacted errors, not anonymous -output. +Phase 2 adds an in-process aggregate-report boundary: + +- an HMAC-SHA256 content ID bound to the exact source bytes; +- domain-separated HMAC IDs emitted only for endpoint values that repeat; +- occurrence counts by IP version, address scope, and IANA port-number range; +- explicit duplicate-group and extra-occurrence counts; +- deterministic, sorted-key JSON for the public report and receipt; +- an in-report Python runtime profile binding stdlib endpoint semantics; +- a SHA-256 digest of the canonical public report bytes, computed outside the + report so the digest does not refer to itself. + +The new API returns no raw address, canonical endpoint, secret key, or unkeyed +digest of the source. It has no file, network, or process API. + +### Privacy and threat boundary + +Parsed `Endpoint` and `EndpointCorpus` objects still retain raw canonical IP +addresses, and `EndpointCorpus` includes an ordinary SHA-256 source digest. +Treat parser models as sensitive: do not log, serialize, publish, or attach +them to issues when they came from a real corpus. Use the privacy-report API +when output must cross that boundary. + +Privacy reports are **pseudonymized aggregates, not anonymous or +irreversible data**. HMAC-SHA256 makes offline enumeration impractical only +while a high-entropy key remains secret. Reusing a key intentionally makes +the exact source content and repeated endpoint groups linkable across reports. +Anyone who can submit chosen corpora to a report-generation service can test +candidates online, so access to that service is part of the key boundary. +The output also reveals source size, physical-line count, category totals, +duplicate equality, and duplicate frequency. Those signals may identify a +small or otherwise recognizable corpus. + +The receipt's report SHA-256 is content integrity metadata, not a signature, +proof of origin, or substitute for an authenticated publication channel. +Python cannot reliably zeroize the caller's input, key, or intermediate +objects. The caller owns key generation, access control, versioning, rotation, +retention, and process isolation. Never hard-code or commit a production key. ## Quick check @@ -46,25 +72,56 @@ python3 -m venv .venv .venv/bin/python -m unittest discover -s tests -v ``` -The parser is a library boundary for now: +The parser remains available for trusted local inspection: ```python from netveil import parse_corpus corpus = parse_corpus(b"192.0.2.10:443\n[2001:db8::10]:8443\n") print(corpus.unique_count) -print(corpus.source_sha256) ``` The addresses above are IETF documentation ranges, not live fixtures. +Build a public aggregate and deterministic receipt with a secret key of at +least 32 bytes: + +```python +import secrets + +from netveil import build_privacy_receipt + +# For a real workflow, load a managed secret instead of generating an +# ephemeral key or embedding one in source code. +key = secrets.token_bytes(32) +receipt = build_privacy_receipt( + b"192.0.2.10:443\n192.0.2.10:443\n", + pseudonymization_key=key, +) +public_bytes = receipt.canonical_json_bytes() +``` + +With one key, equivalent endpoint spellings canonicalize before their +duplicate-group ID is calculated. Rotating the key changes the cryptographic +IDs. Source-content and duplicate-group IDs have different HMAC domains and +different typed prefixes, so they cannot be confused semantically. + +Report bytes are deterministic for the same inputs and the same embedded +runtime profile. The profile records the Python implementation and exact +version because stdlib `ipaddress` parsing and classification can change +between Python releases; cross-profile byte identity is not claimed. + +The exact framing, domains, ordering, port buckets, and canonicalization +profile are specified in +[docs/privacy-protocol.md](docs/privacy-protocol.md). + ## Intended direction The next incremental releases will add: -- aggregate duplicate, range, and schema-quality reports; -- irreversible, domain-separated anonymisation for reportable aggregates; -- content-addressed audit receipts and reproducible CLI evidence; +- richer schema-quality aggregates without weakening the privacy boundary; +- reproducible CLI evidence; +- source-derived diagrams, output examples, and short demonstrations; - installed-artifact guards preserving the no-network boundary as reporting layers are added. diff --git a/SECURITY.md b/SECURITY.md index e18e8d8..cadfbee 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,9 +2,10 @@ ## Supported state -Netveil is in rehabilitation phase 1. It ships an offline parser library but -no command-line scanner, network probe, or stable release. Security fixes -target the default branch and the latest open rehabilitation pull request. +Netveil is in rehabilitation phase 2. It ships an offline parser and +privacy-preserving aggregate-report library, but no command-line scanner, +network probe, or stable release. Security fixes target the default branch and +the latest open rehabilitation pull request. ## Data handling @@ -13,10 +14,36 @@ third-party endpoint inventories in an issue or pull request. Use IETF documentation ranges or clearly synthetic special-use addresses when demonstrating the parser. -Phase-1 `Endpoint` and `EndpointCorpus` objects retain raw canonical addresses. -They are not anonymized and must not be logged, published, or attached to an -issue when created from a real corpus. Redacted exceptions protect error -messages only; irreversible report identifiers are a future phase. +`Endpoint` and `EndpointCorpus` objects retain raw canonical addresses, and +the corpus model retains an unkeyed source SHA-256. They must not be logged, +published, or attached to an issue when created from a real corpus. Redacted +exceptions protect error messages only. + +`build_privacy_report` and `build_privacy_receipt` keep those raw values out of +their returned models. Their source-content and duplicate-group identifiers +use separate versioned HMAC-SHA256 domains and typed prefixes. The secret key +must be exact bytes containing at least 32 bytes. + +The public report binds the Python implementation, exact interpreter version, +and its use of stdlib `ipaddress` semantics. Compare report bytes only within +the same runtime profile; scope classification can change between Python +releases. + +These reports are pseudonymized, not anonymous. HMAC resists offline guessing +only while a high-entropy key remains secret. It does not prevent guessing by +an actor who can submit chosen corpora to a report-generation service, so +access to such a service must also be restricted. A reused key permits +equality linkage across reports. Published output exposes aggregate scope and +port counts, source and line sizes, and the equality and frequency of +duplicated endpoint values. Assess those disclosures against the size and +sensitivity of the input before publishing. + +The receipt's SHA-256 binds the canonical public report bytes but is neither a +signature nor proof of provenance. Python cannot guarantee zeroization of the +source bytes, HMAC key, or intermediate canonical values. Callers must manage +key creation, storage, access, versioning, rotation, retirement, and process +isolation. Never place a production pseudonymization key in source control, +logs, command history, report metadata, or issue attachments. The historical repository contains unverified public endpoint strings. Their presence does not grant permission to connect to, probe, or test those systems. diff --git a/docs/privacy-protocol.md b/docs/privacy-protocol.md new file mode 100644 index 0000000..5b4a6bc --- /dev/null +++ b/docs/privacy-protocol.md @@ -0,0 +1,125 @@ +# Privacy report protocol v1 + +This document makes the byte-level Netveil v1 privacy-report contract +reproducible. It describes pseudonymization and aggregation, not anonymization. + +## Input boundary + +`build_privacy_report` and `build_privacy_receipt` accept: + +- an exact `bytes` endpoint corpus accepted by `parse_corpus`; and +- an exact `bytes` pseudonymization key containing at least 32 bytes. + +Parser size, line-count, UTF-8, syntax, and canonicalization bounds apply +unchanged. The report layer has no path, file, network, DNS, or process API. + +## Typed HMAC identifiers + +Both identifiers use: + +```text +HMAC-SHA256(key, domain || uint64be(value_length) || value) +``` + +`uint64be` is an unsigned eight-byte big-endian integer. Digest text is the 64 +lowercase hexadecimal characters returned by `HMAC.hexdigest()`. + +The exact, stable domains and values are: + +| Identifier | Typed prefix | Domain bytes as a Python literal | Value | +|---|---|---|---| +| source content | `nvs1_` | `b"netveil\x00hmac-sha256-pseudonymization\x00v1\x00source-content\x00"` | exact source bytes | +| duplicate group | `nvd1_` | `b"netveil\x00hmac-sha256-pseudonymization\x00v1\x00duplicate-group\x00"` | canonical endpoint encoded as ASCII | + +The HMAC domains cryptographically separate the purposes. The distinct typed +prefixes also prevent a source ID and duplicate-group ID from being +interpreted as the same semantic type. + +An endpoint ID is emitted only when that canonical endpoint occurs at least +twice. Equivalent IPv6 spellings are canonicalized before grouping. Duplicate +groups are ordered lexicographically by typed ID, never by raw endpoint or +input order. + +## Aggregate semantics + +Every categorical count is an endpoint-occurrence count, not a unique-value +count. Zero-count categories remain present. + +The fixed v1 scope labels are `documentation`, `global`, `private`, +`loopback`, `link_local`, `multicast`, `shared`, `site_local`, `unspecified`, +and `reserved`. Classification is inherited from the fail-closed parser. + +Port buckets are fixed for v1: + +- `system_1_1023`: ports 1 through 1023; +- `registered_1024_49151`: ports 1024 through 49151; +- `dynamic_49152_65535`: ports 49152 through 65535. + +`extra_occurrences` is the sum of `occurrences - 1` across duplicate groups. +`unique_endpoints + extra_occurrences` therefore equals +`endpoint_occurrences`. + +The public report also includes exact source byte size and parser physical-line +count. It never includes an ordinary digest of the source. + +## Canonical JSON and receipt binding + +`netveil.sorted-keys-json.v1` is the following restricted JSON profile: + +- UTF-8-compatible ASCII output from `ensure_ascii=True`; +- object keys sorted lexicographically at every level; +- compact `,` and `:` separators; +- no NaN or infinity; +- no trailing newline. + +The v1 schema contains integers, strings, objects, and arrays only, so number +or Unicode normalization ambiguity is not present. + +The report binds the runtime profile that supplied endpoint parsing, +canonicalization, and scope classification: + +- schema `netveil.python-runtime.v1`; +- `sys.implementation.name`; +- the exact Python `major.minor.micro` version; and +- endpoint semantics `python-stdlib-ipaddress`. + +The same payload and key are byte-deterministic under the same runtime profile. +Python's stdlib `ipaddress` classifications have changed between interpreter +versions, so reports with different runtime profiles are not claimed to be +byte-identical. The profile is inside the report-digest boundary rather than +being ambient metadata. + +The receipt calculates: + +```text +report_sha256 = SHA256(canonical_public_report_bytes) +``` + +It then embeds both the report object and that lowercase digest. The digest +input is the standalone public report, which has no digest field; this avoids +self-reference. The digest is content-addressing metadata, not a signature or +proof of provenance. + +The public identifiers declare +`netveil.hmac-sha256-pseudonymization.v1`; the report and receipt declare +`netveil.aggregate-report.v1` and `netveil.aggregate-receipt.v1`. Any +incompatible change to domains, framing, canonicalization, aggregation, or +schema requires a new protocol/schema version. + +## Threat boundary + +HMAC prevents offline candidate testing only while the key remains secret. It +does not prevent guessing by someone who can submit chosen corpora to a report +generation oracle. Restrict access to both the key and any service that uses +it. + +Reusing a key deliberately exposes equality of exact source content and +duplicate groups across reports. Counts, sizes, categories, equality, and +frequency remain public and may identify a small corpus. Key rotation changes +the typed cryptographic IDs for independently generated keys, but aggregate +patterns can still permit inference. + +Python cannot guarantee zeroization of input bytes, keys, canonical endpoint +strings, or HMAC intermediates. Callers own key entropy, storage, access, +versioning, rotation, retirement, memory/process isolation, and output +disclosure decisions. diff --git a/pyproject.toml b/pyproject.toml index 87ff32f..90f9479 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "netveil-audit" -version = "0.1.0" -description = "Offline fail-closed parser for network-endpoint corpora" +version = "0.2.0" +description = "Offline privacy-preserving audits for network-endpoint corpora" readme = "README.md" requires-python = ">=3.11" license = "MIT" diff --git a/src/netveil/__init__.py b/src/netveil/__init__.py index c9c833c..f2b3536 100644 --- a/src/netveil/__init__.py +++ b/src/netveil/__init__.py @@ -13,15 +13,41 @@ EndpointParseErrorCode, parse_corpus, ) +from netveil.privacy import ( + CANONICAL_JSON_PROTOCOL, + MIN_PSEUDONYMIZATION_KEY_BYTES, + PRIVACY_RECEIPT_SCHEMA, + PRIVACY_REPORT_SCHEMA, + PSEUDONYMIZATION_PROTOCOL, + RUNTIME_PROFILE_SCHEMA, + DuplicateGroup, + PrivacyReceipt, + PrivacyReport, + RuntimeProfile, + build_privacy_receipt, + build_privacy_report, +) __all__ = [ + "CANONICAL_JSON_PROTOCOL", "MAX_INPUT_BYTES", "MAX_PHYSICAL_LINES", + "MIN_PSEUDONYMIZATION_KEY_BYTES", + "PRIVACY_RECEIPT_SCHEMA", + "PRIVACY_REPORT_SCHEMA", + "PSEUDONYMIZATION_PROTOCOL", + "RUNTIME_PROFILE_SCHEMA", + "DuplicateGroup", "Endpoint", "EndpointCorpus", "EndpointParseError", "EndpointParseErrorCode", "EndpointScope", "IPVersion", + "PrivacyReceipt", + "PrivacyReport", + "RuntimeProfile", + "build_privacy_receipt", + "build_privacy_report", "parse_corpus", ] diff --git a/src/netveil/privacy.py b/src/netveil/privacy.py new file mode 100644 index 0000000..882db9a --- /dev/null +++ b/src/netveil/privacy.py @@ -0,0 +1,517 @@ +"""Privacy-preserving aggregate reports for endpoint corpora. + +This module deliberately keeps raw endpoint models on the private side of the +report boundary. Public models contain only keyed identifiers and aggregate +counts. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import sys +from collections import Counter +from dataclasses import dataclass +from typing import Final, Self, final + +from netveil.model import EndpointCorpus, EndpointScope, IPVersion +from netveil.parser import MAX_INPUT_BYTES, MAX_PHYSICAL_LINES, parse_corpus + +MIN_PSEUDONYMIZATION_KEY_BYTES: Final = 32 +PRIVACY_REPORT_SCHEMA: Final = "netveil.aggregate-report.v1" +PRIVACY_RECEIPT_SCHEMA: Final = "netveil.aggregate-receipt.v1" +PSEUDONYMIZATION_PROTOCOL: Final = "netveil.hmac-sha256-pseudonymization.v1" +CANONICAL_JSON_PROTOCOL: Final = "netveil.sorted-keys-json.v1" +RUNTIME_PROFILE_SCHEMA: Final = "netveil.python-runtime.v1" + +_SOURCE_CONTENT_DOMAIN: Final = ( + b"netveil\x00hmac-sha256-pseudonymization\x00v1\x00source-content\x00" +) +_DUPLICATE_GROUP_DOMAIN: Final = ( + b"netveil\x00hmac-sha256-pseudonymization\x00v1\x00duplicate-group\x00" +) +_SOURCE_CONTENT_ID_PREFIX: Final = "nvs1_" +_DUPLICATE_GROUP_ID_PREFIX: Final = "nvd1_" +_HEX_DIGITS: Final = frozenset("0123456789abcdef") +_MIN_CANONICAL_ENDPOINT_BYTES: Final = len(b"[::]:1") + +_IP_VERSION_LABELS: Final = ("ipv4", "ipv6") +_SCOPE_LABELS: Final = tuple(scope.value for scope in EndpointScope) +_PORT_BUCKET_LABELS: Final = ( + "system_1_1023", + "registered_1024_49151", + "dynamic_49152_65535", +) + +CountPairs = tuple[tuple[str, int], ...] + + +@final +@dataclass(frozen=True, slots=True, init=False) +class RuntimeProfile: + """The runtime whose stdlib defines endpoint parsing and classification.""" + + schema: str + python_implementation: str + python_version: str + endpoint_semantics: str + + def __new__(cls) -> Self: + """Block construction outside the report factory.""" + + raise TypeError("RuntimeProfile objects are created by build_privacy_report") + + +@final +@dataclass(frozen=True, slots=True, init=False) +class DuplicateGroup: + """One pseudonymous endpoint value that occurs more than once.""" + + group_id: str + occurrences: int + + def __new__(cls) -> Self: + """Block construction outside the validated report factory.""" + + raise TypeError("DuplicateGroup objects are created by build_privacy_report") + + @property + def extra_occurrences(self) -> int: + """Return occurrences beyond the first instance.""" + + return self.occurrences - 1 + + +@final +@dataclass(frozen=True, slots=True, init=False) +class PrivacyReport: + """Immutable public aggregates with no raw endpoint or source digest.""" + + source_content_id: str + runtime_profile: RuntimeProfile + source_bytes: int + physical_line_count: int + endpoint_count: int + unique_endpoint_count: int + ip_version_counts: CountPairs + scope_counts: CountPairs + port_bucket_counts: CountPairs + duplicate_groups: tuple[DuplicateGroup, ...] + + def __new__(cls) -> Self: + """Block construction outside the validated report factory.""" + + raise TypeError("PrivacyReport objects are created by build_privacy_report") + + @property + def duplicate_group_count(self) -> int: + """Return the number of distinct endpoint values with duplicates.""" + + return len(self.duplicate_groups) + + @property + def duplicate_occurrence_count(self) -> int: + """Return total endpoint occurrences beyond each group's first.""" + + return sum(group.extra_occurrences for group in self.duplicate_groups) + + def canonical_json_bytes(self) -> bytes: + """Serialize the public report using Netveil's canonical JSON profile.""" + + return _canonical_json_bytes(_report_document(self)) + + +@final +@dataclass(frozen=True, slots=True, init=False) +class PrivacyReceipt: + """A report plus a digest of the report's canonical public bytes.""" + + report: PrivacyReport + report_sha256: str + + def __new__(cls) -> Self: + """Block construction outside the validated receipt factory.""" + + raise TypeError("PrivacyReceipt objects are created by build_privacy_receipt") + + def canonical_json_bytes(self) -> bytes: + """Serialize the receipt without including its digest input recursively.""" + + document = { + "canonicalization": CANONICAL_JSON_PROTOCOL, + "protocol": PSEUDONYMIZATION_PROTOCOL, + "report": _report_document(self.report), + "report_digest": { + "algorithm": "sha256", + "value": self.report_sha256, + }, + "schema": PRIVACY_RECEIPT_SCHEMA, + } + return _canonical_json_bytes(document) + + +def _canonical_json_bytes(document: object) -> bytes: + return json.dumps( + document, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def _count_document(counts: CountPairs) -> dict[str, int]: + return dict(counts) + + +def _report_document(report: PrivacyReport) -> dict[str, object]: + return { + "canonicalization": CANONICAL_JSON_PROTOCOL, + "counts": { + "endpoint_occurrences": report.endpoint_count, + "physical_lines": report.physical_line_count, + "source_bytes": report.source_bytes, + "unique_endpoints": report.unique_endpoint_count, + }, + "duplicates": { + "extra_occurrences": report.duplicate_occurrence_count, + "group_count": report.duplicate_group_count, + "groups": [ + { + "extra_occurrences": group.extra_occurrences, + "id": group.group_id, + "occurrences": group.occurrences, + } + for group in report.duplicate_groups + ], + }, + "endpoint_occurrences_by_ip_version": _count_document(report.ip_version_counts), + "endpoint_occurrences_by_port_bucket": _count_document( + report.port_bucket_counts + ), + "endpoint_occurrences_by_scope": _count_document(report.scope_counts), + "protocol": PSEUDONYMIZATION_PROTOCOL, + "runtime": { + "endpoint_semantics": report.runtime_profile.endpoint_semantics, + "python_implementation": report.runtime_profile.python_implementation, + "python_version": report.runtime_profile.python_version, + "schema": report.runtime_profile.schema, + }, + "schema": PRIVACY_REPORT_SCHEMA, + "source_content_id": report.source_content_id, + } + + +def _keyed_identifier( + key: bytes, + *, + domain: bytes, + value: bytes, + prefix: str, +) -> str: + framed_value = len(value).to_bytes(8, byteorder="big") + value + digest = hmac.new(key, domain + framed_value, hashlib.sha256).hexdigest() + return prefix + digest + + +def _validate_pseudonymization_key(key: bytes) -> None: + if type(key) is not bytes: + raise TypeError("pseudonymization_key must be exact bytes") + if len(key) < MIN_PSEUDONYMIZATION_KEY_BYTES: + raise ValueError( + "pseudonymization_key must contain at least " + f"{MIN_PSEUDONYMIZATION_KEY_BYTES} bytes" + ) + + +def _validate_identifier(value: str, *, prefix: str, field: str) -> None: + if type(value) is not str: + raise TypeError(f"{field} must be an exact str") + suffix = value.removeprefix(prefix) + if ( + not value.startswith(prefix) + or len(suffix) != hashlib.sha256().digest_size * 2 + or any(character not in _HEX_DIGITS for character in suffix) + ): + raise ValueError(f"{field} must be a valid typed HMAC-SHA256 identifier") + + +def _validate_exact_nonnegative_int(value: int, *, field: str) -> None: + if type(value) is not int: + raise TypeError(f"{field} must be an exact int") + if value < 0: + raise ValueError(f"{field} must be non-negative") + + +def _validate_count_pairs( + counts: CountPairs, + *, + field: str, + labels: tuple[str, ...], + expected_total: int, +) -> None: + if type(counts) is not tuple: + raise TypeError(f"{field} must be an exact tuple") + for item in counts: + if type(item) is not tuple or len(item) != 2: + raise TypeError(f"{field} entries must be exact two-item tuples") + label, count = item + if type(label) is not str: + raise TypeError(f"{field} labels must be exact strings") + _validate_exact_nonnegative_int(count, field=f"{field} count") + if tuple(label for label, _ in counts) != labels: + raise ValueError(f"{field} labels or order are invalid") + if sum(count for _, count in counts) != expected_total: + raise ValueError(f"{field} must sum to endpoint_count") + + +def _create_duplicate_group(*, group_id: str, occurrences: int) -> DuplicateGroup: + _validate_identifier( + group_id, + prefix=_DUPLICATE_GROUP_ID_PREFIX, + field="group_id", + ) + _validate_exact_nonnegative_int(occurrences, field="occurrences") + if occurrences < 2: + raise ValueError("duplicate group occurrences must be at least two") + + group: DuplicateGroup = object.__new__(DuplicateGroup) + object.__setattr__(group, "group_id", group_id) + object.__setattr__(group, "occurrences", occurrences) + return group + + +def _create_runtime_profile() -> RuntimeProfile: + version = sys.version_info + profile: RuntimeProfile = object.__new__(RuntimeProfile) + object.__setattr__(profile, "schema", RUNTIME_PROFILE_SCHEMA) + object.__setattr__(profile, "python_implementation", sys.implementation.name) + object.__setattr__( + profile, + "python_version", + f"{version.major}.{version.minor}.{version.micro}", + ) + object.__setattr__( + profile, + "endpoint_semantics", + "python-stdlib-ipaddress", + ) + return profile + + +def _validate_runtime_profile(profile: RuntimeProfile) -> None: + if type(profile) is not RuntimeProfile: + raise TypeError("runtime_profile must come from the validated factory") + if profile != _create_runtime_profile(): + raise ValueError("runtime_profile does not match the active runtime") + + +def _create_privacy_report( + *, + source_content_id: str, + runtime_profile: RuntimeProfile, + source_bytes: int, + physical_line_count: int, + endpoint_count: int, + unique_endpoint_count: int, + ip_version_counts: CountPairs, + scope_counts: CountPairs, + port_bucket_counts: CountPairs, + duplicate_groups: tuple[DuplicateGroup, ...], +) -> PrivacyReport: + _validate_identifier( + source_content_id, + prefix=_SOURCE_CONTENT_ID_PREFIX, + field="source_content_id", + ) + _validate_runtime_profile(runtime_profile) + for field, value in ( + ("source_bytes", source_bytes), + ("physical_line_count", physical_line_count), + ("endpoint_count", endpoint_count), + ("unique_endpoint_count", unique_endpoint_count), + ): + _validate_exact_nonnegative_int(value, field=field) + if endpoint_count == 0: + raise ValueError("endpoint_count must be positive") + if source_bytes == 0: + raise ValueError("source_bytes must be positive") + if source_bytes > MAX_INPUT_BYTES: + raise ValueError("source_bytes exceeds the parser input bound") + if physical_line_count > MAX_PHYSICAL_LINES: + raise ValueError("physical_line_count exceeds the parser line bound") + if not 1 <= unique_endpoint_count <= endpoint_count: + raise ValueError("unique_endpoint_count must be in 1..endpoint_count") + if physical_line_count < endpoint_count: + raise ValueError("physical_line_count cannot be smaller than endpoint_count") + minimum_source_bytes = ( + endpoint_count * _MIN_CANONICAL_ENDPOINT_BYTES + physical_line_count - 1 + ) + if source_bytes < minimum_source_bytes: + raise ValueError("source_bytes is too small for the aggregate line counts") + + _validate_count_pairs( + ip_version_counts, + field="ip_version_counts", + labels=_IP_VERSION_LABELS, + expected_total=endpoint_count, + ) + _validate_count_pairs( + scope_counts, + field="scope_counts", + labels=_SCOPE_LABELS, + expected_total=endpoint_count, + ) + _validate_count_pairs( + port_bucket_counts, + field="port_bucket_counts", + labels=_PORT_BUCKET_LABELS, + expected_total=endpoint_count, + ) + if type(duplicate_groups) is not tuple: + raise TypeError("duplicate_groups must be an exact tuple") + if any(type(group) is not DuplicateGroup for group in duplicate_groups): + raise TypeError("every duplicate group must come from the validated factory") + for group in duplicate_groups: + _validate_identifier( + group.group_id, + prefix=_DUPLICATE_GROUP_ID_PREFIX, + field="group_id", + ) + _validate_exact_nonnegative_int(group.occurrences, field="occurrences") + if group.occurrences < 2: + raise ValueError("duplicate group occurrences must be at least two") + group_ids = tuple(group.group_id for group in duplicate_groups) + if group_ids != tuple(sorted(group_ids)) or len(group_ids) != len(set(group_ids)): + raise ValueError("duplicate_groups must have unique IDs in sorted order") + if len(duplicate_groups) > unique_endpoint_count: + raise ValueError("duplicate_group_count cannot exceed unique_endpoint_count") + expected_duplicate_count = endpoint_count - unique_endpoint_count + if ( + sum(group.extra_occurrences for group in duplicate_groups) + != expected_duplicate_count + ): + raise ValueError("duplicate_groups do not match aggregate uniqueness counts") + + report: PrivacyReport = object.__new__(PrivacyReport) + object.__setattr__(report, "source_content_id", source_content_id) + object.__setattr__(report, "runtime_profile", runtime_profile) + object.__setattr__(report, "source_bytes", source_bytes) + object.__setattr__(report, "physical_line_count", physical_line_count) + object.__setattr__(report, "endpoint_count", endpoint_count) + object.__setattr__(report, "unique_endpoint_count", unique_endpoint_count) + object.__setattr__(report, "ip_version_counts", ip_version_counts) + object.__setattr__(report, "scope_counts", scope_counts) + object.__setattr__(report, "port_bucket_counts", port_bucket_counts) + object.__setattr__(report, "duplicate_groups", duplicate_groups) + return report + + +def _create_privacy_receipt(report: PrivacyReport) -> PrivacyReceipt: + if type(report) is not PrivacyReport: + raise TypeError("report must come from build_privacy_report") + + report_sha256 = hashlib.sha256(report.canonical_json_bytes()).hexdigest() + receipt: PrivacyReceipt = object.__new__(PrivacyReceipt) + object.__setattr__(receipt, "report", report) + object.__setattr__(receipt, "report_sha256", report_sha256) + return receipt + + +def _port_bucket(port: int) -> str: + if port <= 1_023: + return "system_1_1023" + if port <= 49_151: + return "registered_1024_49151" + return "dynamic_49152_65535" + + +def _count_pairs(labels: tuple[str, ...], observed: Counter[str]) -> CountPairs: + return tuple((label, observed[label]) for label in labels) + + +def _report_from_corpus( + corpus: EndpointCorpus, + *, + pseudonymization_key: bytes, + payload: bytes, +) -> PrivacyReport: + version_counts: Counter[str] = Counter() + scope_counts: Counter[str] = Counter() + port_counts: Counter[str] = Counter() + canonical_counts: Counter[str] = Counter() + + for endpoint in corpus.endpoints: + version_counts["ipv6" if endpoint.version is IPVersion.IPV6 else "ipv4"] += 1 + scope_counts[endpoint.scope.value] += 1 + port_counts[_port_bucket(endpoint.port)] += 1 + canonical_counts[endpoint.canonical] += 1 + + duplicate_groups = tuple( + sorted( + ( + _create_duplicate_group( + group_id=_keyed_identifier( + pseudonymization_key, + domain=_DUPLICATE_GROUP_DOMAIN, + value=canonical.encode("ascii"), + prefix=_DUPLICATE_GROUP_ID_PREFIX, + ), + occurrences=occurrences, + ) + for canonical, occurrences in canonical_counts.items() + if occurrences > 1 + ), + key=lambda group: group.group_id, + ) + ) + return _create_privacy_report( + source_content_id=_keyed_identifier( + pseudonymization_key, + domain=_SOURCE_CONTENT_DOMAIN, + value=payload, + prefix=_SOURCE_CONTENT_ID_PREFIX, + ), + runtime_profile=_create_runtime_profile(), + source_bytes=corpus.source_bytes, + physical_line_count=corpus.physical_line_count, + endpoint_count=corpus.endpoint_count, + unique_endpoint_count=corpus.unique_count, + ip_version_counts=_count_pairs(_IP_VERSION_LABELS, version_counts), + scope_counts=_count_pairs(_SCOPE_LABELS, scope_counts), + port_bucket_counts=_count_pairs(_PORT_BUCKET_LABELS, port_counts), + duplicate_groups=duplicate_groups, + ) + + +def build_privacy_report( + payload: bytes, + *, + pseudonymization_key: bytes, +) -> PrivacyReport: + """Parse exact bytes locally and return only keyed IDs and aggregates.""" + + if type(payload) is not bytes: + raise TypeError("payload must be exact bytes") + _validate_pseudonymization_key(pseudonymization_key) + corpus = parse_corpus(payload) + return _report_from_corpus( + corpus, + pseudonymization_key=pseudonymization_key, + payload=payload, + ) + + +def build_privacy_receipt( + payload: bytes, + *, + pseudonymization_key: bytes, +) -> PrivacyReceipt: + """Build a public report and bind its canonical bytes in one pass.""" + + report = build_privacy_report( + payload, + pseudonymization_key=pseudonymization_key, + ) + return _create_privacy_receipt(report) diff --git a/tests/test_privacy.py b/tests/test_privacy.py new file mode 100644 index 0000000..bcf7fed --- /dev/null +++ b/tests/test_privacy.py @@ -0,0 +1,597 @@ +from __future__ import annotations + +import hashlib +import json +import socket +import subprocess +import sys +import unittest +from dataclasses import FrozenInstanceError +from typing import Any, cast +from unittest.mock import patch + +from netveil import ( + CANONICAL_JSON_PROTOCOL, + MAX_INPUT_BYTES, + MAX_PHYSICAL_LINES, + MIN_PSEUDONYMIZATION_KEY_BYTES, + PRIVACY_RECEIPT_SCHEMA, + PRIVACY_REPORT_SCHEMA, + PSEUDONYMIZATION_PROTOCOL, + RUNTIME_PROFILE_SCHEMA, + DuplicateGroup, + EndpointParseError, + EndpointParseErrorCode, + PrivacyReceipt, + PrivacyReport, + RuntimeProfile, + build_privacy_receipt, + build_privacy_report, +) +from netveil.privacy import ( + _DUPLICATE_GROUP_DOMAIN, + _SOURCE_CONTENT_DOMAIN, + _create_duplicate_group, + _create_privacy_receipt, + _create_privacy_report, + _create_runtime_profile, + _keyed_identifier, +) + +_KEY_A = bytes(range(32)) +_KEY_B = bytes(range(32, 64)) +_PAYLOAD = ( + b"# Synthetic special-use ranges only\n" + b"192.0.2.1:80\n" + b"192.0.2.1:80\n" + b"[2001:db8::1]:443\n" + b"[2001:0DB8:0:0:0:0:0:1]:443\n" + b"10.0.0.1:1024\n" + b"127.0.0.1:49152\n" +) +_RAW_VALUES = ( + "192.0.2.1:80", + "[2001:db8::1]:443", + "[2001:0DB8:0:0:0:0:0:1]:443", + "10.0.0.1:1024", + "127.0.0.1:49152", +) + + +def _json(payload: bytes) -> dict[str, Any]: + document = json.loads(payload) + if not isinstance(document, dict): + raise TypeError("expected a JSON object") + return cast(dict[str, Any], document) + + +class PrivacyReportTests(unittest.TestCase): + def test_builds_expected_aggregates_without_network_or_process_calls(self) -> None: + with ( + patch.object(socket, "socket", side_effect=AssertionError("network")), + patch.object( + socket, + "getaddrinfo", + side_effect=AssertionError("resolution"), + ), + patch.object(subprocess, "Popen", side_effect=AssertionError("process")), + patch.object(subprocess, "run", side_effect=AssertionError("process")), + ): + report = build_privacy_report( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + + document = _json(report.canonical_json_bytes()) + self.assertEqual(document["schema"], PRIVACY_REPORT_SCHEMA) + self.assertEqual(document["protocol"], PSEUDONYMIZATION_PROTOCOL) + self.assertEqual(document["canonicalization"], CANONICAL_JSON_PROTOCOL) + self.assertEqual( + document["runtime"], + { + "endpoint_semantics": "python-stdlib-ipaddress", + "python_implementation": sys.implementation.name, + "python_version": ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + ), + "schema": RUNTIME_PROFILE_SCHEMA, + }, + ) + self.assertEqual( + document["counts"], + { + "endpoint_occurrences": 6, + "physical_lines": 7, + "source_bytes": len(_PAYLOAD), + "unique_endpoints": 4, + }, + ) + self.assertEqual( + document["endpoint_occurrences_by_ip_version"], + {"ipv4": 4, "ipv6": 2}, + ) + self.assertEqual( + document["endpoint_occurrences_by_port_bucket"], + { + "dynamic_49152_65535": 1, + "registered_1024_49151": 1, + "system_1_1023": 4, + }, + ) + scopes = document["endpoint_occurrences_by_scope"] + self.assertEqual(scopes["documentation"], 4) + self.assertEqual(scopes["private"], 1) + self.assertEqual(scopes["loopback"], 1) + self.assertEqual(sum(scopes.values()), 6) + self.assertEqual( + document["duplicates"]["extra_occurrences"], + 2, + ) + self.assertEqual(document["duplicates"]["group_count"], 2) + groups = document["duplicates"]["groups"] + self.assertEqual([group["occurrences"] for group in groups], [2, 2]) + self.assertEqual([group["extra_occurrences"] for group in groups], [1, 1]) + self.assertEqual( + [group["id"] for group in groups], + sorted(group["id"] for group in groups), + ) + + def test_output_and_failures_do_not_reveal_raw_values_or_source_sha(self) -> None: + receipt = build_privacy_receipt( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + rendered = ( + receipt.canonical_json_bytes().decode("ascii") + + repr(receipt) + + repr(receipt.report) + ) + for raw_value in _RAW_VALUES: + self.assertNotIn(raw_value, rendered) + self.assertNotIn(hashlib.sha256(_PAYLOAD).hexdigest(), rendered) + self.assertNotIn(_KEY_A.hex(), rendered) + + invalid_value = "invalid-sensitive-value" + with self.assertRaises(EndpointParseError) as raised: + build_privacy_report( + f"{invalid_value}:443\n".encode(), + pseudonymization_key=_KEY_A, + ) + self.assertEqual( + raised.exception.code, + EndpointParseErrorCode.INVALID_ADDRESS, + ) + self.assertNotIn(invalid_value, str(raised.exception)) + self.assertNotIn(invalid_value, repr(raised.exception)) + + def test_report_and_receipt_are_deterministic_canonical_json(self) -> None: + first = build_privacy_receipt( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + second = build_privacy_receipt( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + self.assertEqual(first, second) + self.assertEqual(first.canonical_json_bytes(), second.canonical_json_bytes()) + + report_bytes = first.report.canonical_json_bytes() + receipt_document = _json(first.canonical_json_bytes()) + self.assertEqual(receipt_document["schema"], PRIVACY_RECEIPT_SCHEMA) + self.assertEqual( + receipt_document["report"], + _json(report_bytes), + ) + self.assertEqual( + first.report_sha256, + hashlib.sha256(report_bytes).hexdigest(), + ) + self.assertEqual( + receipt_document["report_digest"], + {"algorithm": "sha256", "value": first.report_sha256}, + ) + self.assertNotIn("report_digest", _json(report_bytes)) + + for canonical_bytes in (report_bytes, first.canonical_json_bytes()): + self.assertFalse(canonical_bytes.endswith(b"\n")) + self.assertEqual( + canonical_bytes, + json.dumps( + json.loads(canonical_bytes), + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii"), + ) + + def test_key_rotation_and_domain_separation_change_typed_identifiers(self) -> None: + first = build_privacy_report( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + rotated = build_privacy_report( + _PAYLOAD, + pseudonymization_key=_KEY_B, + ) + self.assertNotEqual(first.source_content_id, rotated.source_content_id) + self.assertTrue(first.source_content_id.startswith("nvs1_")) + self.assertTrue( + all(group.group_id.startswith("nvd1_") for group in first.duplicate_groups) + ) + self.assertNotEqual( + {group.group_id for group in first.duplicate_groups}, + {group.group_id for group in rotated.duplicate_groups}, + ) + + same_value = b"same-framed-value" + source_id = _keyed_identifier( + _KEY_A, + domain=_SOURCE_CONTENT_DOMAIN, + value=same_value, + prefix="nvs1_", + ) + duplicate_id = _keyed_identifier( + _KEY_A, + domain=_DUPLICATE_GROUP_DOMAIN, + value=same_value, + prefix="nvd1_", + ) + self.assertEqual( + source_id, + "nvs1_1835ed652ea947a88125064fad24a25b4cd3faa2c4f3e05878ce2bfe8fdd66c5", + ) + self.assertEqual( + duplicate_id, + "nvd1_c5422fb8589253f084b16b227ba8fc7a8d1e2f69ec257f93a10f4a2544941750", + ) + self.assertNotEqual( + source_id.removeprefix("nvs1_"), duplicate_id.removeprefix("nvd1_") + ) + self.assertNotEqual(source_id, duplicate_id) + + def test_equivalent_endpoints_have_stable_duplicate_ids_across_corpora( + self, + ) -> None: + first = build_privacy_report( + b"[2001:db8::1]:443\n[2001:0DB8:0:0:0:0:0:1]:443\n", + pseudonymization_key=_KEY_A, + ) + second = build_privacy_report( + b"# another source\n[2001:db8::1]:443\n[2001:db8::1]:443\n", + pseudonymization_key=_KEY_A, + ) + self.assertNotEqual(first.source_content_id, second.source_content_id) + self.assertEqual( + first.duplicate_groups[0].group_id, + second.duplicate_groups[0].group_id, + ) + + def test_single_endpoint_report_has_no_duplicate_identifier(self) -> None: + report = build_privacy_report( + b"192.0.2.1:443\n", + pseudonymization_key=_KEY_A, + ) + self.assertEqual(report.duplicate_groups, ()) + self.assertEqual(report.duplicate_group_count, 0) + self.assertEqual(report.duplicate_occurrence_count, 0) + + def test_exact_types_key_length_and_parser_bounds_fail_closed(self) -> None: + class BytesSubclass(bytes): + pass + + for payload in ( + "192.0.2.1:443\n", + bytearray(b"192.0.2.1:443\n"), + BytesSubclass(b"192.0.2.1:443\n"), + ): + with ( + self.subTest(payload_type=type(payload).__name__), + self.assertRaisesRegex(TypeError, "^payload must be exact bytes$"), + ): + build_privacy_report( + payload, # type: ignore[arg-type] + pseudonymization_key=_KEY_A, + ) + + for key in ( + "x" * MIN_PSEUDONYMIZATION_KEY_BYTES, + bytearray(_KEY_A), + BytesSubclass(_KEY_A), + ): + with ( + self.subTest(key_type=type(key).__name__), + self.assertRaisesRegex( + TypeError, + "^pseudonymization_key must be exact bytes$", + ), + ): + build_privacy_report( + b"192.0.2.1:443\n", + pseudonymization_key=key, # type: ignore[arg-type] + ) + + short_key = b"sensitive-but-too-short" + with self.assertRaisesRegex( + ValueError, + "^pseudonymization_key must contain at least 32 bytes$", + ) as raised: + build_privacy_report( + b"192.0.2.1:443\n", + pseudonymization_key=short_key, + ) + self.assertNotIn(short_key.decode(), str(raised.exception)) + + with self.assertRaises(EndpointParseError) as oversized: + build_privacy_report( + b"x" * (MAX_INPUT_BYTES + 1), + pseudonymization_key=_KEY_A, + ) + self.assertEqual( + oversized.exception.code, + EndpointParseErrorCode.INPUT_TOO_LARGE, + ) + + def test_public_models_are_frozen_and_direct_construction_is_blocked(self) -> None: + for model, message in ( + ( + RuntimeProfile, + "RuntimeProfile objects are created by build_privacy_report", + ), + ( + DuplicateGroup, + "DuplicateGroup objects are created by build_privacy_report", + ), + ( + PrivacyReport, + "PrivacyReport objects are created by build_privacy_report", + ), + ( + PrivacyReceipt, + "PrivacyReceipt objects are created by build_privacy_receipt", + ), + ): + with ( + self.subTest(model=model.__name__), + self.assertRaisesRegex(TypeError, f"^{message}$"), + ): + model() + + receipt = build_privacy_receipt( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + with self.assertRaises(FrozenInstanceError): + receipt.report.endpoint_count = 0 # type: ignore[misc] + with self.assertRaises(FrozenInstanceError): + receipt.report.duplicate_groups[0].occurrences = 99 # type: ignore[misc] + with self.assertRaises(FrozenInstanceError): + receipt.report_sha256 = "0" * 64 # type: ignore[misc] + + +class PrivacyModelInvariantTests(unittest.TestCase): + def setUp(self) -> None: + self.report = build_privacy_report( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + self.valid: dict[str, object] = { + "source_content_id": self.report.source_content_id, + "runtime_profile": self.report.runtime_profile, + "source_bytes": self.report.source_bytes, + "physical_line_count": self.report.physical_line_count, + "endpoint_count": self.report.endpoint_count, + "unique_endpoint_count": self.report.unique_endpoint_count, + "ip_version_counts": self.report.ip_version_counts, + "scope_counts": self.report.scope_counts, + "port_bucket_counts": self.report.port_bucket_counts, + "duplicate_groups": self.report.duplicate_groups, + } + + def create(self, **changes: object) -> PrivacyReport: + values = self.valid | changes + return _create_privacy_report( + source_content_id=cast(str, values["source_content_id"]), + runtime_profile=cast(RuntimeProfile, values["runtime_profile"]), + source_bytes=cast(int, values["source_bytes"]), + physical_line_count=cast(int, values["physical_line_count"]), + endpoint_count=cast(int, values["endpoint_count"]), + unique_endpoint_count=cast(int, values["unique_endpoint_count"]), + ip_version_counts=cast( + tuple[tuple[str, int], ...], + values["ip_version_counts"], + ), + scope_counts=cast( + tuple[tuple[str, int], ...], + values["scope_counts"], + ), + port_bucket_counts=cast( + tuple[tuple[str, int], ...], + values["port_bucket_counts"], + ), + duplicate_groups=cast( + tuple[DuplicateGroup, ...], + values["duplicate_groups"], + ), + ) + + def test_duplicate_group_factory_rejects_invalid_models(self) -> None: + with self.assertRaisesRegex(TypeError, "^group_id must be an exact str$"): + _create_duplicate_group(group_id=cast(str, b"x"), occurrences=2) + for group_id in ( + "wrong_" + "0" * 64, + "nvd1_short", + "nvd1_" + "G" * 64, + ): + with ( + self.subTest(group_id=group_id), + self.assertRaisesRegex(ValueError, "^group_id must be a valid"), + ): + _create_duplicate_group(group_id=group_id, occurrences=2) + with self.assertRaisesRegex(TypeError, "^occurrences must be an exact int$"): + _create_duplicate_group( + group_id="nvd1_" + "0" * 64, + occurrences=True, + ) + for occurrences in (-1, 0, 1): + with ( + self.subTest(occurrences=occurrences), + self.assertRaises(ValueError), + ): + _create_duplicate_group( + group_id="nvd1_" + "0" * 64, + occurrences=occurrences, + ) + + def test_report_factory_rejects_invalid_identifiers_and_scalar_counts( + self, + ) -> None: + with self.assertRaisesRegex(TypeError, "^source_content_id must be"): + self.create(source_content_id=cast(str, b"x")) + with self.assertRaisesRegex(ValueError, "^source_content_id must be"): + self.create(source_content_id="nvs1_" + "z" * 64) + with self.assertRaisesRegex(TypeError, "^runtime_profile must come"): + self.create(runtime_profile=object()) + + forged_profile: RuntimeProfile = object.__new__(RuntimeProfile) + object.__setattr__(forged_profile, "schema", RUNTIME_PROFILE_SCHEMA) + object.__setattr__( + forged_profile, + "python_implementation", + sys.implementation.name, + ) + object.__setattr__(forged_profile, "python_version", "0.0.0") + object.__setattr__( + forged_profile, + "endpoint_semantics", + "python-stdlib-ipaddress", + ) + with self.assertRaisesRegex(ValueError, "^runtime_profile does not match"): + self.create(runtime_profile=forged_profile) + + self.assertEqual(self.report.runtime_profile, _create_runtime_profile()) + + for field in ( + "source_bytes", + "physical_line_count", + "endpoint_count", + "unique_endpoint_count", + ): + with ( + self.subTest(field=field), + self.assertRaisesRegex(TypeError, f"^{field} must be an exact int$"), + ): + self.create(**{field: True}) + with self.assertRaisesRegex(ValueError, "^source_bytes must be non-negative$"): + self.create(source_bytes=-1) + with self.assertRaisesRegex(ValueError, "^source_bytes must be positive$"): + self.create(source_bytes=0) + with self.assertRaisesRegex(ValueError, "^source_bytes exceeds"): + self.create(source_bytes=MAX_INPUT_BYTES + 1) + with self.assertRaisesRegex(ValueError, "^physical_line_count exceeds"): + self.create(physical_line_count=MAX_PHYSICAL_LINES + 1) + with self.assertRaisesRegex(ValueError, "^endpoint_count must be positive$"): + self.create(endpoint_count=0, unique_endpoint_count=0) + for unique_count in (0, self.report.endpoint_count + 1): + with ( + self.subTest(unique_count=unique_count), + self.assertRaisesRegex(ValueError, "^unique_endpoint_count must be"), + ): + self.create(unique_endpoint_count=unique_count) + with self.assertRaisesRegex(ValueError, "^physical_line_count cannot"): + self.create(physical_line_count=self.report.endpoint_count - 1) + with self.assertRaisesRegex(ValueError, "^source_bytes is too small"): + self.create(source_bytes=1) + + def test_report_factory_rejects_invalid_count_tables(self) -> None: + with self.assertRaisesRegex(TypeError, "^ip_version_counts must be"): + self.create(ip_version_counts=list(self.report.ip_version_counts)) + with self.assertRaisesRegex( + TypeError, + "^ip_version_counts entries must be", + ): + self.create(ip_version_counts=(("ipv4", 4), ["ipv6", 2])) + with self.assertRaisesRegex( + TypeError, + "^ip_version_counts entries must be", + ): + self.create(ip_version_counts=(("ipv4", 4, 0), ("ipv6", 2))) + with self.assertRaisesRegex( + TypeError, + "^ip_version_counts labels must be", + ): + self.create(ip_version_counts=((cast(str, b"ipv4"), 4), ("ipv6", 2))) + with self.assertRaisesRegex( + TypeError, + "^ip_version_counts count must be an exact int$", + ): + self.create(ip_version_counts=(("ipv4", True), ("ipv6", 5))) + with self.assertRaisesRegex( + ValueError, + "^ip_version_counts count must be non-negative$", + ): + self.create(ip_version_counts=(("ipv4", -1), ("ipv6", 7))) + with self.assertRaisesRegex( + ValueError, + "^ip_version_counts labels or order are invalid$", + ): + self.create(ip_version_counts=(("ipv6", 2), ("ipv4", 4))) + with self.assertRaisesRegex( + ValueError, + "^ip_version_counts must sum to endpoint_count$", + ): + self.create(ip_version_counts=(("ipv4", 3), ("ipv6", 2))) + + def test_report_factory_rejects_invalid_duplicate_groups(self) -> None: + with self.assertRaisesRegex(TypeError, "^duplicate_groups must be"): + self.create(duplicate_groups=list(self.report.duplicate_groups)) + with self.assertRaisesRegex(TypeError, "^every duplicate group must"): + self.create(duplicate_groups=(object(),)) + + first, second = self.report.duplicate_groups + invalid_id: DuplicateGroup = object.__new__(DuplicateGroup) + object.__setattr__(invalid_id, "group_id", "invalid") + object.__setattr__(invalid_id, "occurrences", 2) + with self.assertRaisesRegex(ValueError, "^group_id must be a valid"): + self.create(duplicate_groups=(invalid_id,)) + + invalid_type: DuplicateGroup = object.__new__(DuplicateGroup) + object.__setattr__(invalid_type, "group_id", first.group_id) + object.__setattr__(invalid_type, "occurrences", True) + with self.assertRaisesRegex(TypeError, "^occurrences must be an exact int$"): + self.create(duplicate_groups=(invalid_type,)) + + too_small: DuplicateGroup = object.__new__(DuplicateGroup) + object.__setattr__(too_small, "group_id", first.group_id) + object.__setattr__(too_small, "occurrences", 1) + with self.assertRaisesRegex(ValueError, "^duplicate group occurrences"): + self.create(duplicate_groups=(too_small,)) + + with self.assertRaisesRegex(ValueError, "^duplicate_groups must have"): + self.create(duplicate_groups=(second, first)) + with self.assertRaisesRegex(ValueError, "^duplicate_groups must have"): + self.create(duplicate_groups=(first, first)) + + too_many_groups = ( + _create_duplicate_group(group_id="nvd1_" + "0" * 64, occurrences=2), + _create_duplicate_group(group_id="nvd1_" + "1" * 64, occurrences=2), + _create_duplicate_group(group_id="nvd1_" + "2" * 64, occurrences=3), + ) + with self.assertRaisesRegex(ValueError, "^duplicate_group_count cannot"): + self.create( + unique_endpoint_count=2, + duplicate_groups=too_many_groups, + ) + with self.assertRaisesRegex(ValueError, "^duplicate_groups do not match"): + self.create(duplicate_groups=()) + + def test_receipt_factory_rejects_nonfactory_report(self) -> None: + with self.assertRaisesRegex(TypeError, "^report must come from"): + _create_privacy_receipt(cast(PrivacyReport, object())) + + +if __name__ == "__main__": + unittest.main() From aaf24377dbe4bae0ba6020af577f711fb5100edf Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:26:42 +0000 Subject: [PATCH 4/7] Prevent parser inputs leaking through exception chains --- README.md | 6 ++++++ SECURITY.md | 5 ++++- src/netveil/parser.py | 11 +++++++---- tests/test_parser.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9f9776d..7198155 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,12 @@ Python cannot reliably zeroize the caller's input, key, or intermediate objects. The caller owns key generation, access control, versioning, rotation, retention, and process isolation. Never hard-code or commit a production key. +Parser failures use bounded codes and line numbers. Their library-created +payload and exception context do not retain a rejected endpoint or undecodable +corpus. Caller-owned input objects, ambient exception context, traceback frame +locals, debuggers, and crash dump tooling remain outside that boundary and can +still retain sensitive bytes. + ## Quick check ```bash diff --git a/SECURITY.md b/SECURITY.md index cadfbee..cd231a8 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -17,7 +17,10 @@ demonstrating the parser. `Endpoint` and `EndpointCorpus` objects retain raw canonical addresses, and the corpus model retains an unkeyed source SHA-256. They must not be logged, published, or attached to an issue when created from a real corpus. Redacted -exceptions protect error messages only. +parser exceptions protect the library-owned payload and do not create a +`UnicodeDecodeError` or address-parser context retaining rejected input. +Caller objects, ambient exception context, traceback frame locals, debuggers, +and crash dump tooling remain outside that boundary. `build_privacy_report` and `build_privacy_receipt` keep those raw values out of their returned models. Their source-content and duplicate-group identifiers diff --git a/src/netveil/parser.py b/src/netveil/parser.py index 88d333e..4013b01 100644 --- a/src/netveil/parser.py +++ b/src/netveil/parser.py @@ -4,6 +4,7 @@ import ipaddress import re +from contextlib import suppress from enum import Enum from typing import NoReturn @@ -100,9 +101,10 @@ def _parse_endpoint(raw_line: str, *, line_number: int) -> Endpoint: raw_port = match.group("port") if "%" in raw_address: _fail(EndpointParseErrorCode.INVALID_ADDRESS, line_number=line_number) - try: + parsed_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None + with suppress(ValueError): parsed_address = ipaddress.ip_address(raw_address) - except ValueError: + if parsed_address is None: _fail(EndpointParseErrorCode.INVALID_ADDRESS, line_number=line_number) if (ipv6_match is not None) != isinstance( @@ -126,9 +128,10 @@ def parse_corpus(payload: bytes) -> EndpointCorpus: _fail(EndpointParseErrorCode.INPUT_TOO_LARGE) if b"\r" in payload.replace(b"\r\n", b""): _fail(EndpointParseErrorCode.INVALID_LINE_ENDING) - try: + text: str | None = None + with suppress(UnicodeDecodeError): text = payload.decode("utf-8", errors="strict") - except UnicodeDecodeError: + if text is None: _fail(EndpointParseErrorCode.INVALID_UTF8) if any(character in _NON_LF_LINE_SEPARATORS for character in text): _fail(EndpointParseErrorCode.INVALID_LINE_ENDING) diff --git a/tests/test_parser.py b/tests/test_parser.py index 0c04e04..fe532e2 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -3,6 +3,7 @@ import hashlib import ipaddress import socket +import traceback import unittest from unittest.mock import patch @@ -196,6 +197,23 @@ def test_rejects_invalid_addresses_without_echoing_them(self) -> None: forbidden=forbidden, ) + def test_invalid_address_creates_no_library_exception_context(self) -> None: + payload = b"PRIVATE-ENDPOINT-VALUE:443\n" + + with self.assertRaises(EndpointParseError) as raised: + parse_corpus(payload) + + error = raised.exception + rendered = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + self.assertEqual(error.code, EndpointParseErrorCode.INVALID_ADDRESS) + self.assertIsNone(error.__context__) + self.assertIsNone(error.__cause__) + self.assertNotIn("PRIVATE-ENDPOINT-VALUE", repr(error)) + self.assertNotIn("PRIVATE-ENDPOINT-VALUE", rendered) + self.assertNotIn("AddressValueError", rendered) + def test_rejects_invalid_and_noncanonical_ports(self) -> None: cases = ( (b"192.0.2.1:0\n", EndpointParseErrorCode.INVALID_PORT), @@ -240,6 +258,23 @@ def test_rejects_invalid_utf8_and_non_lf_line_endings(self) -> None: line_number=None, ) + def test_invalid_utf8_creates_no_library_exception_context(self) -> None: + payload = b"PRIVATE-CORPUS-\xff-DO-NOT-ECHO" + + with self.assertRaises(EndpointParseError) as raised: + parse_corpus(payload) + + error = raised.exception + rendered = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + self.assertEqual(error.code, EndpointParseErrorCode.INVALID_UTF8) + self.assertIsNone(error.__context__) + self.assertIsNone(error.__cause__) + self.assertNotIn(repr(payload), repr(error)) + self.assertNotIn("PRIVATE-CORPUS", rendered) + self.assertNotIn("UnicodeDecodeError", rendered) + def test_enforces_resource_bounds(self) -> None: self.assert_rejected( b"x" * (MAX_INPUT_BYTES + 1), From 3cc6bcef1dc5a960e69f92c705f4b3f264726760 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:12:58 +0000 Subject: [PATCH 5/7] Guard installed receipt execution --- .gitignore | 4 + MANIFEST.in | 17 +- README.md | 316 ++++-- SECURITY.md | 192 +++- docs/artifact-boundary.md | 155 +++ docs/assets/architecture.svg | 56 + docs/cli-contract.md | 141 +++ docs/privacy-protocol.md | 14 +- pyproject.toml | 8 +- requirements-dev.txt | 14 + scripts/netveil-audit | 296 +++++ src/netveil/cli.py | 361 ++++++ src/netveil_bootstrap.py | 540 +++++++++ tests/fixtures/launcher_bootstrap_fixture.py | 7 + tests/test_bootstrap.py | 1040 ++++++++++++++++++ tests/test_cli.py | 696 ++++++++++++ tests/test_launcher.py | 568 ++++++++++ tests/test_packaging.py | 38 + tests/test_visuals.py | 23 + tools/__init__.py | 1 + tools/render_architecture.py | 300 +++++ 21 files changed, 4608 insertions(+), 179 deletions(-) create mode 100644 docs/artifact-boundary.md create mode 100644 docs/assets/architecture.svg create mode 100644 docs/cli-contract.md create mode 100644 requirements-dev.txt create mode 100755 scripts/netveil-audit create mode 100644 src/netveil/cli.py create mode 100644 src/netveil_bootstrap.py create mode 100644 tests/fixtures/launcher_bootstrap_fixture.py create mode 100644 tests/test_bootstrap.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_launcher.py create mode 100644 tests/test_visuals.py create mode 100644 tools/__init__.py create mode 100755 tools/render_architecture.py diff --git a/.gitignore b/.gitignore index 98729d4..4ab2b56 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ __pycache__/ .ruff_cache/ .mypy_cache/ .venv/ +.demo-venv/ .coverage *.egg-info/ dist/ @@ -12,3 +13,6 @@ build/ # Raw endpoint corpora may contain third-party infrastructure and stay local. *.endpoints data/raw/ +corpus.txt +receipt.json +receipt.key diff --git a/MANIFEST.in b/MANIFEST.in index 9c97a7a..63844e1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,17 @@ +include LICENSE +include README.md include SECURITY.md -include docs/privacy-protocol.md +include requirements-dev.txt +include pyproject.toml +include scripts/netveil-audit +recursive-include docs *.md +recursive-include docs/assets *.svg *.json *.cast *.gif +recursive-include src *.py py.typed +recursive-include tests *.py +recursive-include tools *.py +global-exclude *.py[cod] +global-exclude .coverage +global-exclude .DS_Store +global-exclude *~ +prune build +prune dist diff --git a/README.md b/README.md index 7198155..11cb918 100644 --- a/README.md +++ b/README.md @@ -1,143 +1,227 @@ # Netveil -Netveil is being rebuilt as an offline, fail-closed audit tool for -network-endpoint corpora with an explicit privacy boundary. - -## Current status - -This repository is at **rehabilitation phase 2**. Its former working tree -contained unverified third-party IP address and port lists collected in 2022. -Those live endpoint lists have been removed from the current tree: they were -not suitable fixtures, did not include provenance or consent, and must not be -interpreted as working services. - -The old values remain in Git history until a separately coordinated history -rewrite. Do not use them for connection attempts, availability checks, or -security testing. - -The first layer is a dependency-free, offline parser with: - -- strict UTF-8, line-ending, IP-address, and port validation; -- canonical IPv4 and bracketed IPv6 representations; -- deterministic duplicate detection across equivalent IPv6 spellings; -- explicit network-scope classification, including documentation ranges; -- content binding to the SHA-256 digest of the exact input bytes; -- bounded input and safe errors that never repeat endpoint values. - -It does not import a networking client, resolve DNS, or open sockets. - -Phase 2 adds an in-process aggregate-report boundary: - -- an HMAC-SHA256 content ID bound to the exact source bytes; -- domain-separated HMAC IDs emitted only for endpoint values that repeat; -- occurrence counts by IP version, address scope, and IANA port-number range; -- explicit duplicate-group and extra-occurrence counts; -- deterministic, sorted-key JSON for the public report and receipt; -- an in-report Python runtime profile binding stdlib endpoint semantics; -- a SHA-256 digest of the canonical public report bytes, computed outside the - report so the digest does not refer to itself. - -The new API returns no raw address, canonical endpoint, secret key, or unkeyed -digest of the source. It has no file, network, or process API. - -### Privacy and threat boundary - -Parsed `Endpoint` and `EndpointCorpus` objects still retain raw canonical IP -addresses, and `EndpointCorpus` includes an ordinary SHA-256 source digest. -Treat parser models as sensitive: do not log, serialize, publish, or attach -them to issues when they came from a real corpus. Use the privacy-report API -when output must cross that boundary. - -Privacy reports are **pseudonymized aggregates, not anonymous or -irreversible data**. HMAC-SHA256 makes offline enumeration impractical only -while a high-entropy key remains secret. Reusing a key intentionally makes -the exact source content and repeated endpoint groups linkable across reports. -Anyone who can submit chosen corpora to a report-generation service can test -candidates online, so access to that service is part of the key boundary. -The output also reveals source size, physical-line count, category totals, -duplicate equality, and duplicate frequency. Those signals may identify a -small or otherwise recognizable corpus. - -The receipt's report SHA-256 is content integrity metadata, not a signature, -proof of origin, or substitute for an authenticated publication channel. -Python cannot reliably zeroize the caller's input, key, or intermediate -objects. The caller owns key generation, access control, versioning, rotation, -retention, and process isolation. Never hard-code or commit a production key. - -Parser failures use bounded codes and line numbers. Their library-created -payload and exception context do not retain a rejected endpoint or undecodable -corpus. Caller-owned input objects, ambient exception context, traceback frame -locals, debuggers, and crash dump tooling remain outside that boundary and can -still retain sensitive bytes. - -## Quick check +Netveil turns a private local corpus of IP endpoints into a deterministic, +pseudonymized audit receipt without resolving, connecting to, or probing any +endpoint. + +The unusual part is not the JSON. The wheel-installed command starts through +an isolated static launcher, binds selected installed files to installer- +derived distribution metadata, source-loads a closed package inventory from +already-pinned bytes, and omits raw endpoints, caller paths, key material, and +unkeyed corpus hashes from its successful receipt and Python-handled Netveil +diagnostics. + +> **Status:** 0.3.0 is a release candidate on the rehabilitation pull-request +> branch, not a published stable release. The guarded command has been +> exercised on CPython 3.12.3 / Linux x86-64. See the exact +> [artifact boundary](docs/artifact-boundary.md) before relying on it. + +## What it produces + +Input stays local: + +```text +# IETF documentation ranges only +192.0.2.10:443 +192.0.2.10:443 +[2001:db8::10]:8443 +``` + +The public receipt contains fixed-schema counts, runtime semantics, typed +HMAC-SHA256 identifiers for exact source content and repeated endpoint groups, +and a digest of the canonical public report. It does not contain +`192.0.2.10`, `2001:db8::10`, the key, or an ordinary SHA-256 of the input. + +Receipt disclosure is still a decision: sizes, category totals, duplicate +equality, and duplicate frequency remain visible. Pseudonymized does not mean +anonymous. + +## Installed workflow + +Use an exact wheel in a fresh standard POSIX virtual environment. Verify its +published digest through an authenticated channel before installation; the +runtime guard is not a signature. ```bash -python3 -m venv .venv -.venv/bin/python -m pip install -e . -.venv/bin/python -m unittest discover -s tests -v +WHEEL=dist/netveil_audit-0.3.0-py3-none-any.whl +EXPECTED_SHA256='' +printf '%s %s\n' "$EXPECTED_SHA256" "$WHEEL" \ + | sha256sum --check --strict + +python3 -m venv .demo-venv +.demo-venv/bin/python -m pip install --no-index --no-deps "$WHEEL" +.demo-venv/bin/netveil-audit --version ``` -The parser remains available for trusted local inspection: +Create a synthetic demonstration corpus and a private ephemeral key: -```python -from netveil import parse_corpus +```bash +umask 077 -corpus = parse_corpus(b"192.0.2.10:443\n[2001:db8::10]:8443\n") -print(corpus.unique_count) +printf '%s\n' \ + '# IETF documentation ranges only' \ + '192.0.2.10:443' \ + '192.0.2.10:443' \ + '[2001:db8::10]:8443' > corpus.txt + +.demo-venv/bin/python - <<'PY' +import os +import secrets + +descriptor = os.open( + "receipt.key", + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, +) +with os.fdopen(descriptor, "wb") as stream: + stream.write(secrets.token_bytes(32)) +PY + +.demo-venv/bin/netveil-audit \ + receipt corpus.txt \ + --key-file receipt.key > receipt.json ``` -The addresses above are IETF documentation ranges, not live fixtures. +The key is exact binary input: do not create a production key with `echo`, +hard-code it, reuse a corpus as a key, or commit it. Netveil requires 32–4096 +bytes, effective-user ownership, exactly one hard link, and owner-only +permissions. The complete file and exit-code contract is in +[docs/cli-contract.md](docs/cli-contract.md). + +## How the guard works + +![Code-derived Netveil installed execution boundary](docs/assets/architecture.svg) + +_Selected launcher flags, parser/CLI bounds, and the closed module inventory +are extracted from source by `tools/render_architecture.py`; the remaining +labels document the reviewed design, and the test suite pins the complete +renderer output._ + +```text +installed polyglot launcher + │ /bin/sh -> sibling python -I -E -S -B + ▼ +startup-profile + installed-RECORD drift check + │ compile checked netveil_bootstrap.py bytes + ▼ +closed package inventory + bounded descriptor reads + │ compile pinned netveil.* source bytes in memory + ▼ +local corpus + owner-only key + │ parse -> canonicalize -> aggregate -> domain-separated HMAC + ▼ +canonical pseudonymized receipt on stdout +``` -Build a public aggregate and deterministic receipt with a secret key of at -least 32 bytes: +The static launcher closes the earlier `PYTHONPATH`, `sitecustomize`, and +unchecked-bootstrap-`pyc` gap of a generated console wrapper. The package +finder rejects unknown `netveil.*` modules and does not execute package +bytecode. -```python -import secrets +The remaining trust boundary is explicit: + +- the installed launcher is already executing before it can check itself; +- `/bin/sh`, the sibling CPython interpreter, stdlib, installer, OS, + filesystem, standard virtual-environment layout, and installed + `.dist-info/RECORD` are trusted; +- the installed `RECORD` is mutable, unsigned, and may differ from the wheel + archive's original record after installer rewrites; coordinated + code-plus-`RECORD` changes are accepted; +- the file checks are individually race-aware but do not create a + transactionally atomic snapshot against a hostile concurrent mutator; +- the checks detect uncoordinated artifact drift; they do not prove publisher + identity, provenance, freshness, or an uncompromised host. + +Read [docs/artifact-boundary.md](docs/artifact-boundary.md) for the layer-by- +layer threat model and non-claims. + +The receipt does not embed the wheel digest, source commit, distribution +version, or evidence identity. Preserve the fresh-wheel verification manifest +beside any published receipt when artifact provenance matters. + +## Privacy protocol + +The parser is dependency-free and fail-closed: + +- strict UTF-8 and unambiguous IPv4/bracketed-IPv6 endpoint syntax; +- 8 MiB and 100,000-physical-line bounds; +- deterministic canonicalization and duplicate detection; +- explicit address-scope and IANA port-range counts; +- redacted parser failures that retain a bounded code and optional line number, + not the rejected endpoint. + +The report protocol uses separate versioned HMAC domains for source content +and repeated endpoint groups. Key rotation changes both typed identifiers. +Equivalent IPv6 spellings canonicalize before equality grouping. Canonical +JSON is byte-deterministic only for the same exact source bytes, key bytes, +code/artifact identity, and embedded Python runtime profile. -from netveil import build_privacy_receipt +[docs/privacy-protocol.md](docs/privacy-protocol.md) specifies the framing, +domains, schemas, ordering, runtime binding, count semantics, and disclosure +limits. -# For a real workflow, load a managed secret instead of generating an -# ephemeral key or embedding one in source code. -key = secrets.token_bytes(32) +## Library boundary + +Trusted local code can use the parser or report builder directly: + +```python +from netveil import build_privacy_receipt, parse_corpus + +payload = b"192.0.2.10:443\n[2001:db8::10]:8443\n" +corpus = parse_corpus(payload) receipt = build_privacy_receipt( - b"192.0.2.10:443\n192.0.2.10:443\n", - pseudonymization_key=key, + payload, + pseudonymization_key=b"\x01" * 32, ) -public_bytes = receipt.canonical_json_bytes() ``` -With one key, equivalent endpoint spellings canonicalize before their -duplicate-group ID is calculated. Rotating the key changes the cryptographic -IDs. Source-content and duplicate-group IDs have different HMAC domains and -different typed prefixes, so they cannot be confused semantically. +This example key is deterministic test material, not production guidance. +Library calls—including an ordinary `import netveil` from the installed +wheel—do not use the installed-command guard. `Endpoint` and `EndpointCorpus` +models retain raw canonical addresses, and the corpus model retains an unkeyed +source SHA-256. Do not log or publish those objects when the input is +sensitive. -Report bytes are deterministic for the same inputs and the same embedded -runtime profile. The profile records the Python implementation and exact -version because stdlib `ipaddress` parsing and classification can change -between Python releases; cross-profile byte identity is not claimed. +## Developer checks -The exact framing, domains, ordering, port buckets, and canonicalization -profile are specified in -[docs/privacy-protocol.md](docs/privacy-protocol.md). +Editable installation is for development only and is expected to fail the +guarded CLI contract. -## Intended direction +```bash +python3 -m venv .venv +.venv/bin/python -m pip install --requirement requirements-dev.txt +.venv/bin/python -m pip install --no-build-isolation --no-deps --editable . + +.venv/bin/ruff check . +.venv/bin/ruff format --check . +.venv/bin/mypy +.venv/bin/coverage run -m unittest discover -s tests +.venv/bin/coverage report -m +``` -The next incremental releases will add: +`requirements-dev.txt` pins the additional Python development packages used +by this repository on CPython 3.12/Linux. It is not a runtime dependency set +or a cross-platform, hash-locked software-supply-chain attestation. -- richer schema-quality aggregates without weakening the privacy boundary; -- reproducible CLI evidence; -- source-derived diagrams, output examples, and short demonstrations; -- installed-artifact guards preserving the no-network boundary as reporting - layers are added. +The current source gate covers the extensionless launcher, bootstrap, CLI, +parser, models, and privacy protocol at 100% statement and branch coverage. +A separate installed-artifact gate builds no trust from the editable +environment: it accepts an exact wheel, installs it into a fresh venv with +`--no-index --no-deps`, and exercises startup injection, bytecode, tamper, +determinism, and redaction cases. -No scanner, proxy checker, or network probe is shipped. +## Deliberate exclusions -## Safety boundary +Netveil is not a scanner, proxy checker, service-discovery client, reachability +tester, anonymizer, key vault, package signature verifier, sandbox, or remote +attestation system. It never grants permission to test systems you do not own +or lack authorization to assess. -Netveil operates on caller-supplied local bytes only and fails closed on -malformed input. Only synthetic or explicitly redistributable fixtures will be -committed. -Testing systems you do not own or lack permission to assess is out of scope. +The repository historically contained unverified third-party endpoint lists +collected in 2022. They were removed from the current tree because they had no +adequate provenance or consent. Their presence in Git history is not evidence +that any service is live and not authorization to connect to it. -See [SECURITY.md](SECURITY.md) for responsible-use and disclosure guidance. +Use only caller-owned data, explicitly redistributable fixtures, or IETF +documentation ranges in examples and issues. See [SECURITY.md](SECURITY.md) +for disclosure and sensitive-data handling guidance. diff --git a/SECURITY.md b/SECURITY.md index cd231a8..f92f4f1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -2,59 +2,145 @@ ## Supported state -Netveil is in rehabilitation phase 2. It ships an offline parser and -privacy-preserving aggregate-report library, but no command-line scanner, -network probe, or stable release. Security fixes target the default branch and -the latest open rehabilitation pull request. - -## Data handling - -Do not submit real credentials, access tokens, private-network inventories, or -third-party endpoint inventories in an issue or pull request. Use IETF -documentation ranges or clearly synthetic special-use addresses when -demonstrating the parser. - -`Endpoint` and `EndpointCorpus` objects retain raw canonical addresses, and -the corpus model retains an unkeyed source SHA-256. They must not be logged, -published, or attached to an issue when created from a real corpus. Redacted -parser exceptions protect the library-owned payload and do not create a -`UnicodeDecodeError` or address-parser context retaining rejected input. -Caller objects, ambient exception context, traceback frame locals, debuggers, -and crash dump tooling remain outside that boundary. - -`build_privacy_report` and `build_privacy_receipt` keep those raw values out of -their returned models. Their source-content and duplicate-group identifiers -use separate versioned HMAC-SHA256 domains and typed prefixes. The secret key -must be exact bytes containing at least 32 bytes. - -The public report binds the Python implementation, exact interpreter version, -and its use of stdlib `ipaddress` semantics. Compare report bytes only within -the same runtime profile; scope classification can change between Python -releases. - -These reports are pseudonymized, not anonymous. HMAC resists offline guessing -only while a high-entropy key remains secret. It does not prevent guessing by -an actor who can submit chosen corpora to a report-generation service, so -access to such a service must also be restricted. A reused key permits -equality linkage across reports. Published output exposes aggregate scope and -port counts, source and line sizes, and the equality and frequency of -duplicated endpoint values. Assess those disclosures against the size and -sensitivity of the input before publishing. - -The receipt's SHA-256 binds the canonical public report bytes but is neither a -signature nor proof of provenance. Python cannot guarantee zeroization of the -source bytes, HMAC key, or intermediate canonical values. Callers must manage -key creation, storage, access, versioning, rotation, retirement, and process -isolation. Never place a production pseudonymization key in source control, -logs, command history, report metadata, or issue attachments. - -The historical repository contains unverified public endpoint strings. Their -presence does not grant permission to connect to, probe, or test those systems. -The current tree intentionally contains none of those values. +Security work targets the default branch and the latest open rehabilitation +pull request. Netveil 0.3.0 is currently a release candidate, not a published +stable release. + +The supported guarded execution path is narrow: install an exact wheel into a +fresh, standard POSIX CPython virtual environment and invoke the installed +`bin/netveil-audit` script. Recorded evidence currently covers CPython 3.12.3 +on Linux x86-64. The library declares Python 3.11 or newer, but equivalent +launcher guarantees are not yet claimed for every interpreter or platform. + +Editable installs, direct source execution, ordinary `import netveil` library +use, `python -m netveil`, Windows, PyPy, Conda, `pip --target`, renamed or +externally symlinked launchers, and non-standard environment layouts are +outside the guarded 0.3.0 command contract. + +## Installed-command trust boundary + +The wheel contains a real POSIX shell/Python polyglot launcher. `/bin/sh` +executes the sibling virtual-environment interpreter with `-I -E -S -B` before +launcher Python code. The launcher checks that isolated, no-site, +environment-ignoring, no-bytecode-write profile; verifies its installed bytes +and `netveil_bootstrap.py` against selected entries in the installed +`.dist-info/RECORD`; and compiles the already-read bootstrap source in memory. + +The bootstrap checks a closed `netveil/` inventory, bounded file reads, +selected distribution metadata, and source hashes. It then installs a closed +finder that compiles pinned source bytes instead of importing source or +bytecode from disk. Preloaded and unknown `netveil.*` modules are rejected. + +These checks reduce startup injection and uncoordinated installation drift. +They are not self-authentication: + +- the launcher is already executing before its own later check; +- `/bin/sh`, sibling CPython, stdlib/import machinery, installer, OS, + filesystem, standard virtual-environment layout, and launcher are trusted; +- installed `.dist-info/RECORD` is mutable, unsigned consistency metadata, + can be rewritten by the installer, and is trusted; +- coherent changes to code and `RECORD` can pass; +- per-file identity rechecks do not form a transactionally atomic snapshot + against a hostile concurrent filesystem mutator; +- a privileged hostile host, kernel, debugger, ptrace client, injected native + code, malicious trusted stdlib, or memory/crash-dump collector is out of + scope. + +Verify a wheel SHA-256 through an authenticated channel before installation. +Netveil does not currently verify a package signature, publisher identity, +provenance attestation, or transparency-log entry. Full details and +unsupported paths are in +[docs/artifact-boundary.md](docs/artifact-boundary.md). + +## Local file handling + +After reading its installed launcher, metadata, and package files for +verification, the receipt workflow opens no caller data paths other than +`CORPUS` and `KEY_FILE`. Both must be direct POSIX regular files opened with +no-follow and nonblocking flags. `O_NOFOLLOW` applies to the final path +component; symlinks in parent directories are followed. The same descriptor's +identity is compared before and after each bounded read. + +The key must contain 32–4096 exact bytes, be owned by the effective user, have +one hard link, be owner-readable, and expose no owner-execute, group, other, or +special permission bits. The corpus cannot be the same inode as the key or a +byte-identical copy. Netveil does not measure key entropy or replace a secret +manager. Never put a production key in source control, logs, shell history, +report metadata, issue attachments, screenshots, or demo recordings. + +Corpus ownership, permission bits, and hard-link count are not restricted. +Input paths remain visible to the operating system and may be visible in a +process listing even though Netveil diagnostics do not repeat them. The caller +also controls stdout redirection, destination permissions, atomic publication, +and retention; Netveil writes receipt bytes to stdout rather than securely +creating an output file. + +The full syntax, size, permission, and exit-code contract is in +[docs/cli-contract.md](docs/cli-contract.md). + +## Privacy boundary + +`Endpoint` and `EndpointCorpus` retain raw canonical addresses. The corpus +model also retains an unkeyed source SHA-256. Treat those library objects, +caller input bytes, traceback frame locals, debuggers, and crash dumps as +sensitive. Redacted library exceptions protect the library-created message +and context; they cannot erase caller-owned objects or ambient exception +state. + +`build_privacy_report`, `build_privacy_receipt`, and the installed `receipt` +command keep raw endpoint strings, secret key bytes, and the ordinary source +digest out of their returned public models. Source-content and duplicate-group +identifiers use separate versioned HMAC-SHA256 domains and typed prefixes. + +Reports are pseudonymized, not anonymous. HMAC resists offline guessing only +while a high-entropy key remains secret. It does not stop an actor who can +submit chosen corpora to a report-generation oracle. Reusing a key deliberately +links exact source content and repeated endpoint groups across reports. +Published output also reveals source size, line count, category totals, +duplicate equality, and duplicate frequency. Review those disclosures before +publication. + +The report records `sys.implementation.name` and numeric Python +`major.minor.micro` whose stdlib `ipaddress` behavior supplied parsing and +classification. It does not hash the interpreter executable or stdlib. +Compare bytes only for the same trusted code artifact and runtime semantics. +Python cannot guarantee zeroization of source bytes, keys, canonical strings, +HMAC intermediates, or allocator copies. + +The receipt's SHA-256 binds canonical public report bytes. It is not a +signature, MAC, proof of origin, or authenticated publication channel. The +receipt does not identify the source commit, wheel digest, distribution +version, or fresh-wheel evidence manifest. The +byte-level protocol and remaining disclosure surface are specified in +[docs/privacy-protocol.md](docs/privacy-protocol.md). + +## Offline and authorization boundary + +Netveil package code contains no intentional DNS, socket, subprocess, or +child-process operation. A publishable installed-artifact evidence bundle must +pair hostile Python environment-variable tests with Linux syscall tracing for +one exact artifact and runtime. That trace is evidence for the recorded run, +not a guarantee about a compromised trusted interpreter, shell, operating +system, or preload mechanism. + +Netveil is not a scanner or reachability tester. Historical Git commits +contain unverified public endpoint strings removed from the current tree. +Their existence does not grant permission to connect to, probe, or test those +systems. Use only caller-owned data, explicitly redistributable fixtures, or +IETF documentation ranges. ## Reporting a vulnerability -Use GitHub's private vulnerability reporting feature for this repository when -available. Include the affected commit, a minimal reproduction with synthetic -data, the expected safety boundary, and the observed behaviour. Do not include -secrets or personal data. +Use GitHub private vulnerability reporting for this repository when available. +Include: + +- the affected commit and exact wheel SHA-256; +- interpreter, OS, install method, and invocation path; +- a minimal reproduction using synthetic data; +- the expected documented boundary and observed behavior; +- whether code and `RECORD` were changed independently or coherently. + +Do not attach real endpoint inventories, private-network layouts, credentials, +access tokens, keys, personal data, memory dumps, or absolute private paths. +If private reporting is unavailable, open a minimal issue that contains no +sensitive reproduction material and asks the maintainer for a secure channel. diff --git a/docs/artifact-boundary.md b/docs/artifact-boundary.md new file mode 100644 index 0000000..3f6bb64 --- /dev/null +++ b/docs/artifact-boundary.md @@ -0,0 +1,155 @@ +# Installed artifact boundary + +Netveil's installed guard reduces accidental drift and Python import-path +ambiguity. It is not package signing, sandboxing, or host attestation. + +## Execution sequence + +The supported wheel installs a real POSIX shell/Python polyglot script rather +than a generated `console_scripts` wrapper: + +```text +wheel-installed netveil-audit + -> /bin/sh + -> sibling venv/bin/python -I -E -S -B + -> static launcher checks startup and installed-RECORD-bound files + -> installed-RECORD-bound netveil_bootstrap.py is compiled from pinned bytes + -> package inventory and source records are checked + -> netveil.* is compiled from pinned source bytes by a closed finder + -> receipt command reads local corpus/key files + -> canonical pseudonymized JSON +``` + +The combined Python flags take effect before launcher Python code: + +- `-I`: isolated mode and safe import path; +- `-E`: ignore Python environment variables; +- `-S`: do not import `site` or `sitecustomize`; +- `-B`: do not write bytecode caches. + +The launcher verifies those flags at runtime. It locates one exact standard +virtual-environment site directory, preflights installed `METADATA` and +`RECORD` as regular files of at most 1,048,576 bytes, checks the installed +script and `netveil_bootstrap.py` against their SHA-256/size entries in the +installed `.dist-info/RECORD`, and compiles the already-read bootstrap bytes +in memory. It never imports the top-level bootstrap by name. + +`importlib.metadata` subsequently parses those trusted installed metadata +files and may reread them. The preflight is a resource bound, not an +authentication step or atomic snapshot. Selected metadata payloads and every +record-bound launcher/bootstrap/package file have the same 1,048,576-byte +per-file runtime ceiling. + +The bootstrap then: + +- requires the exact distribution name and version; +- requires no entry points and the exact `top_level.txt` bytes + `netveil\nnetveil_bootstrap\n`; +- checks selected metadata and every allowed `netveil/` source record; +- opens files with bounded, no-follow descriptor reads and identity rechecks; +- rejects unknown files inside the package directory; +- treats allowed package bytecode as inert and rejects unknown package cache + names; +- refuses a preloaded `netveil` package; +- installs a closed first-position finder; +- compiles only the pinned source bytes already held in memory; +- rejects imports of unlisted `netveil.*` modules. + +## Trust table + +| Layer | Status | +|---|---| +| authenticated wheel SHA-256 obtained before install | external trust root | +| installer and install-time environment | trust root | +| operating system and filesystem semantics | trust root | +| `/bin/sh` and sibling CPython interpreter | trust root | +| CPython stdlib and import machinery | trust root | +| installed static launcher | directly executed trust root | +| standard POSIX virtual-environment layout | required precondition | +| installed `.dist-info/RECORD` | installer-derived mutable consistency input and trust root | +| `netveil_bootstrap.py` bytes | checked against installed `RECORD` before execution | +| `netveil.*` source bytes | checked, pinned, and compiled in memory | +| corpus and key files | sensitive caller-controlled inputs | +| canonical receipt | public pseudonymized output, subject to disclosure review | + +## Non-claims + +The runtime never reads the wheel archive or verifies its SHA-256. It reads +the installer-produced `.dist-info/RECORD`, whose launcher path and bytes can +differ from the wheel archive's original record after installation. +`RECORD` is unsigned. A party able to change code and update it coherently can +make the consistency checks accept those new bytes. The guard therefore does +**not** prove authenticity, publisher identity, provenance, freshness, or +installation from a particular wheel. Verify a published wheel digest through +an authenticated channel before installing it. + +The launcher is already executing when it checks its own current bytes. A +modified launcher can run code before its later mismatch is detected. The +launcher, `/bin/sh`, interpreter, stdlib, OS, filesystem, and `RECORD` remain +inside the trusted computing base. + +The runtime does not hash the sibling Python interpreter, `/bin/sh`, installer, +or stdlib. It does not enforce owner or write-mode policy for installed +artifact files; the launcher checks only that the installed script is a +regular file. The separate release-evidence gate additionally requires the +wheel and installed launcher modes to be owner-executable. A compromised +installer or install-time host can coherently alter installed files and +`RECORD`. + +`-B` prevents bytecode writes; it is not the reason existing bytecode is inert. +The top-level bootstrap is compiled directly from checked source bytes, and +package modules are served by the closed in-memory source finder. Unknown +package cache files are rejected. + +The guard is not a sandbox and does not defend against a privileged hostile +host, kernel compromise, debugger, ptrace, process-memory reader, injected +native code, malicious trusted stdlib, crash-dump collector, or coherently +mutated trust roots. Per-file identity checks do not create a transactionally +atomic snapshot across the whole installation against a hostile concurrent +filesystem mutator. Python cannot guarantee secret-memory zeroization. + +The runtime bootstrap verifies selected distribution records and the closed +`netveil/` package directory. It does not enforce the exact full wheel or sdist +inventory; that is a separate release gate. + +The receipt schema itself contains no wheel digest, source commit, +distribution version, or evidence-manifest identifier. Those provenance +bindings remain external release evidence. + +## Unsupported invocation paths + +The following are deliberately outside the 0.3.0 guarded command contract: + +- editable installs and direct source-checkout execution; +- ordinary `import netveil` library use, including from the installed wheel; +- `python -m netveil`; +- invoking the launcher with ordinary Python flags instead of its shell + handoff; +- Windows, PyPy, Conda, `pip --target`, and non-standard install layouts; +- renamed launcher files or invocation through external symlinks; +- preloaded or embedded `netveil` module state. + +A renamed or expected-name symlink is rejected once Python starts. An external +symlink without the expected adjacent interpreter can fail in the shell before +Netveil controls diagnostics; shell output may then include the invocation +path. Use the installed script in the environment's own `bin/` directory. + +## Evidence standard + +A publishable artifact must be checked from a clean source commit and fresh +wheel installation. The gate must bind: + +- source commit, wheel SHA-256, interpreter version, and exact archive + inventory; +- launcher bytes, mode, shebang, installed `RECORD`, and absence of + `entry_points.txt`; +- isolated startup under hostile Python environment variables; +- inert bootstrap/package bytecode and rejection of unknown package files; +- fail-closed uncoordinated source and metadata tampering; +- deterministic receipt bytes from synthetic IETF documentation ranges; +- absence of raw endpoint and key material in public outputs; +- an offline syscall trace for the exact tested Linux execution. + +Coordinated code-plus-`RECORD` mutations belong in negative trust-boundary +evidence: they must be shown as accepted, not misleadingly presented as +attacks the unsigned guard prevents. diff --git a/docs/assets/architecture.svg b/docs/assets/architecture.svg new file mode 100644 index 0000000..4688844 --- /dev/null +++ b/docs/assets/architecture.svg @@ -0,0 +1,56 @@ + +Netveil installed execution and trust boundary +Code-derived architecture from the installed polyglot launcher through pinned source loading to a pseudonymized receipt. + + + + + +Netveil installed execution boundary +Generated from the 0.3.0 launcher, parser, CLI and bootstrap sources + +1 Trusted startup +/bin/sh → sibling CPython +combined flags: -I -E -S -B +no site hooks • env ignored • safe path +launcher + installed RECORD are trust roots + +2 Artifact drift guard +launcher and bootstrap: installed hash + size +bounded O_NOFOLLOW descriptor reads +identity checked before and after read +installed RECORD is not a signature + +3 Closed source loader +5 allowed modules +pinned source bytes compiled in memory +unknown netveil.* imports rejected +package bytecode never executes + + + +4 Sensitive local inputs +corpus: regular file • ≤ 8 MiB • ≤ 100,000 lines +key: owner-only regular file • 32–4,096 exact bytes +same inode or byte-identical corpus/key rejected +sensitive values omitted from handled output +no DNS, socket or child-process workflow + +5 Public deterministic result +parse → canonicalize → aggregate +domain-separated HMAC-SHA256 identifiers +fixed-schema sorted-key canonical JSON +receipt digest binds the public report +pseudonymized, not anonymous + + + +Exact source inventory +netveil, netveil.cli, netveil.model, netveil.parser, netveil.privacy +Trust-root legend +amber = executes before or supplies the mutable consistency boundary +Verified-byte legend +cyan / blue = checked before compilation by the supported installed path +Reproduce: python3 tools/render_architecture.py --check +No network data or endpoint fixture is used by this diagram + diff --git a/docs/cli-contract.md b/docs/cli-contract.md new file mode 100644 index 0000000..8ca314a --- /dev/null +++ b/docs/cli-contract.md @@ -0,0 +1,141 @@ +# Installed CLI contract + +This document specifies Netveil 0.3.0's local, wheel-installed command. It is +an interface contract, not a claim that mutable Python packaging metadata is a +signature. The exact artifact and trust boundary are documented in +[artifact-boundary.md](artifact-boundary.md). + +## Supported command surface + +The command grammar is deliberately small: + +```text +netveil-audit --help +netveil-audit --version +netveil-audit receipt CORPUS --key-file KEY_FILE +``` + +`--version` is accepted only as the sole argument; mixing it with a receipt, +another option, or a second `--version` is a usage error. Top-level and +`receipt` help remain available through argparse's ordinary `-h`/`--help` +forms. Long options are not abbreviated, and `--key-file` must appear exactly +once in a receipt invocation. + +The supported guarded path is the `netveil-audit` script installed from a +wheel into the standard `bin/` directory of a POSIX CPython virtual +environment. Netveil 0.3.0's recorded execution evidence uses CPython 3.12.3 +on Linux x86-64. The package declares Python 3.11 or newer, but the guarded +launcher is not yet claimed as verified on every such interpreter or on +Windows, PyPy, Conda, editable installs, `pip --target`, or non-standard +environment layouts. + +`python -m netveil` is intentionally unavailable. Running the source checkout, +the launcher through an ordinary non-isolated Python command, or an editable +install is not a substitute for the installed guard. + +## Corpus file + +`CORPUS` must resolve directly to one POSIX regular file. The command opens it +read-only with `O_CLOEXEC`, `O_NOCTTY`, `O_NOFOLLOW`, and `O_NONBLOCK`, then +checks the same descriptor's identity before and after a bounded read. +`O_NOFOLLOW` applies only to the final path component; symlinks in parent +directories are followed. Corpus owner, permission bits, and hard-link count +are not restricted. + +The parser accepts at most 8 MiB and 100,000 physical lines. Input must be +strict UTF-8. Empty lines and lines beginning with `#` are ignored; other lines +must contain an IPv4 literal in `IPv4:port` form or an IPv6 literal in +bracketed `[IPv6]:port` form. Accepted addresses are canonicalized before +grouping. Ports are decimal integers from 1 through 65535 with no leading +zeroes. CRLF and LF are accepted; ambiguous or non-LF Unicode separators are +rejected. + +No address is resolved or contacted. The file can contain private data, so its +path and rejected endpoint text are never repeated in Python-handled Netveil +diagnostics. The input paths remain process arguments and can be visible to the +operating system or process observers. + +## Key file + +`KEY_FILE` is read as exact bytes. It is not decoded, trimmed, or normalized; +a trailing newline is therefore part of the key. The file must satisfy every +condition below: + +- one POSIX regular file, not a symlink, FIFO, socket, or directory; +- 32 through 4096 bytes inclusive; +- owned by the process's effective user ID; +- exactly one hard link; +- readable by the owner; +- no owner execute bit; +- no group or other permission bits; +- no set-user-ID, set-group-ID, or sticky bit. + +Mode `0400` or `0600` is accepted. The corpus and key must not be the same +inode and must not contain byte-identical payloads. Those checks prevent a +corpus from being accidentally reused as its own pseudonymization key; they do +not assess entropy or operate a key-management system. + +## Output + +On success, `receipt` writes one canonical +`netveil.aggregate-receipt.v1` JSON document followed by exactly one LF to +standard output. Standard error is empty. + +For the same exact code/artifact identity, corpus bytes, key bytes, and +embedded Python runtime profile, the output bytes are deterministic. Different +Python micro versions are not promised to be byte-identical because the +runtime profile and stdlib `ipaddress` semantics are inside the receipt's +digest boundary. + +The receipt contains pseudonymized aggregate data, not anonymous data. It +reveals input size, physical-line count, category totals, duplicate equality, +and duplicate frequency. It contains neither raw endpoint strings, key bytes, +nor an unkeyed digest of the corpus. See +[privacy-protocol.md](privacy-protocol.md) for the byte-level schema and HMAC +domains. + +Netveil writes the receipt only to stdout. Shell redirection, downstream pipe +behavior, destination permissions, atomic publication, and retention are the +caller's responsibility. The receipt does not embed the wheel SHA-256, source +commit, distribution version, or fresh-wheel evidence identity. + +## Exit codes + +Diagnostics are stable, one-line, and intentionally omit caller paths and +input values. + +| Exit | Diagnostic or result | +|---:|---| +| 0 | receipt, help, or version completed | +| 2 | `netveil-audit: usage_error` | +| 10 | `netveil-audit: artifact_unverified` or `netveil-audit: platform_unsupported` from the bootstrap | +| 11 | `netveil-audit: corpus_unavailable` | +| 12 | `netveil-audit: key_unavailable` | +| 13 | `netveil-audit: corpus_rejected:[:line=N]` | +| 14 | `netveil-audit: key_rejected` | +| 15 | `netveil-audit: output_failed` | +| 70 | `netveil-audit: internal_error` or `netveil-audit: interrupted` | + +If standard error itself cannot make bounded forward progress, the process +returns 15 and may be unable to emit the complete diagnostic. + +Oversized, non-regular, symlinked, unavailable, or identity-changing corpus +files fail at the file boundary with exit 11. The equivalent key-file cases +use exit 12. A short or policy-insecure key, or a key that is the same inode or +byte-identical payload as the corpus, uses exit 14. Because the command's +8 MiB file read bound runs before parsing, an oversized installed-CLI corpus +does not reach the parser's library-level `input_too_large` code. + +Treat stdout as valid only when the process exits 0. An output failure can +leave a partial stdout prefix before exit 15. Failures in `/bin/sh` or sibling +interpreter execution occur before Netveil's Python error boundary and can use +OS-defined exit codes and shell diagnostics, including an invocation path. + +## Offline scope + +Netveil package code exposes no DNS, socket, subprocess, or child-process path. +A publishable release-evidence bundle must exercise the exact installed +artifact under syscall tracing and hostile Python environment variables. Such +evidence describes the tested Linux run; it is not a mathematical guarantee +about a compromised interpreter, shell, operating system, debugger, or preload +mechanism outside the stated trust boundary. diff --git a/docs/privacy-protocol.md b/docs/privacy-protocol.md index 5b4a6bc..11f71a3 100644 --- a/docs/privacy-protocol.md +++ b/docs/privacy-protocol.md @@ -83,11 +83,11 @@ canonicalization, and scope classification: - the exact Python `major.minor.micro` version; and - endpoint semantics `python-stdlib-ipaddress`. -The same payload and key are byte-deterministic under the same runtime profile. -Python's stdlib `ipaddress` classifications have changed between interpreter -versions, so reports with different runtime profiles are not claimed to be -byte-identical. The profile is inside the report-digest boundary rather than -being ambient metadata. +The same payload and key are byte-deterministic under the same runtime profile +and exact protocol implementation. Python's stdlib `ipaddress` +classifications have changed between interpreter versions, so reports with +different runtime profiles are not claimed to be byte-identical. The profile +is inside the report-digest boundary rather than being ambient metadata. The receipt calculates: @@ -98,7 +98,9 @@ report_sha256 = SHA256(canonical_public_report_bytes) It then embeds both the report object and that lowercase digest. The digest input is the standalone public report, which has no digest field; this avoids self-reference. The digest is content-addressing metadata, not a signature or -proof of provenance. +proof of provenance. The receipt schema does not identify a source commit, +wheel digest, distribution version, interpreter executable hash, or external +evidence manifest. The public identifiers declare `netveil.hmac-sha256-pseudonymization.v1`; the report and receipt declare diff --git a/pyproject.toml b/pyproject.toml index 90f9479..fb0deb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netveil-audit" -version = "0.2.0" +version = "0.3.0" description = "Offline privacy-preserving audits for network-endpoint corpora" readme = "README.md" requires-python = ">=3.11" @@ -16,6 +16,8 @@ dependencies = [] [tool.setuptools] package-dir = {"" = "src"} packages = ["netveil"] +py-modules = ["netveil_bootstrap"] +script-files = ["scripts/netveil-audit"] [tool.setuptools.package-data] netveil = ["py.typed"] @@ -27,11 +29,11 @@ target-version = "py311" [tool.mypy] python_version = "3.11" strict = true -files = ["src", "tests"] +files = ["src", "tests", "tools", "scripts/netveil-audit"] [tool.coverage.run] branch = true -source = ["netveil"] +source = ["src", "scripts"] [tool.coverage.report] fail_under = 99 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..39fd5d5 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,14 @@ +# Exact tool versions used by the repository's local quality gate. +# Runtime dependencies remain empty; this file is development-only. +ast_serialize==0.6.0 +build==1.5.0 +coverage==7.15.2 +librt==0.13.0 +mypy==2.3.0 +mypy_extensions==1.1.0 +packaging==26.2 +pathspec==1.1.1 +pyproject_hooks==1.2.0 +ruff==0.16.0 +setuptools==83.0.0 +typing_extensions==4.16.0 diff --git a/scripts/netveil-audit b/scripts/netveil-audit new file mode 100755 index 0000000..2910244 --- /dev/null +++ b/scripts/netveil-audit @@ -0,0 +1,296 @@ +#!/bin/sh +""":" +case "$0" in + */*) netveil_script_directory=${0%/*} ;; + *) exit 70 ;; +esac +exec "$netveil_script_directory/python" -IESB "$0" "$@" +exit 70 +":""" + +"""Isolated installed-artifact launcher for Netveil.""" + +import base64 +import hashlib +import importlib.metadata +import os +import stat +import sys +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Final, NoReturn + +_DISTRIBUTION_NAME: Final = "netveil-audit" +_DISTRIBUTION_VERSION: Final = "0.3.0" +_BOOTSTRAP_FILE: Final = "netveil_bootstrap.py" +_LAUNCHER_NAME: Final = "netveil-audit" +_MAX_ARTIFACT_FILE_BYTES: Final = 1_048_576 +_ARTIFACT_FAILURE_EXIT: Final = 10 +_OUTPUT_FAILURE_EXIT: Final = 15 +_INTERNAL_FAILURE_EXIT: Final = 70 + + +class _LaunchFailure(Exception): + pass + + +def _fail() -> NoReturn: + raise _LaunchFailure + + +def _startup_is_isolated() -> bool: + flags = sys.flags + return ( + os.name == "posix" + and flags.isolated == 1 + and flags.no_site == 1 + and flags.ignore_environment == 1 + and flags.dont_write_bytecode == 1 + and flags.safe_path is True + and "" not in sys.path + ) + + +def _file_flags() -> int: + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if any(not hasattr(os, name) for name in required): + _fail() + return os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + + +def _identity(status: os.stat_result) -> tuple[int, ...]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_uid, + status.st_gid, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def _read_record_bound( + path: Path, + record: importlib.metadata.PackagePath, +) -> tuple[bytes, tuple[int, ...]]: + file_hash = record.hash + file_size = record.size + if ( + file_hash is None + or file_hash.mode != "sha256" + or type(file_size) is not int + or file_size < 0 + or file_size > _MAX_ARTIFACT_FILE_BYTES + ): + _fail() + + payload, identity = _read_bounded_regular(path) + digest = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()) + digest_text = digest.rstrip(b"=").decode("ascii") + if len(payload) != file_size or digest_text != file_hash.value: + _fail() + return payload, identity + + +def _read_bounded_regular(path: Path) -> tuple[bytes, tuple[int, ...]]: + descriptor = -1 + try: + descriptor = os.open(path, _file_flags()) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size < 0 + or before.st_size > _MAX_ARTIFACT_FILE_BYTES + ): + _fail() + chunks: list[bytes] = [] + observed = 0 + while observed <= before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail() + finally: + if descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + try: + os.close(descriptor) + except OSError: + _fail() + + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail() + return payload, _identity(after) + + +def _site_root(launcher_path: Path) -> Path: + if launcher_path.name != _LAUNCHER_NAME or launcher_path.parent.name != "bin": + _fail() + prefix = launcher_path.parent.parent + python_directory = f"python{sys.version_info.major}.{sys.version_info.minor}" + library_directories = tuple(dict.fromkeys(("lib", sys.platlibdir))) + candidates = tuple( + prefix / library / python_directory / package_directory + for library in library_directories + for package_directory in ("site-packages", "dist-packages") + ) + matches = tuple( + candidate + for candidate in candidates + if (candidate / f"netveil_audit-{_DISTRIBUTION_VERSION}.dist-info").is_dir() + ) + if len(matches) != 1: + _fail() + return matches[0] + + +def _records( + distribution: importlib.metadata.Distribution, +) -> dict[str, importlib.metadata.PackagePath]: + files = distribution.files + if files is None: + _fail() + records: dict[str, importlib.metadata.PackagePath] = {} + for record in files: + raw_path = str(record) + if raw_path in records: + _fail() + records[raw_path] = record + return records + + +def _verified_bootstrap( + launcher_path: Path, +) -> tuple[bytes, Path]: + site_root = _site_root(launcher_path) + dist_info = site_root / f"netveil_audit-{_DISTRIBUTION_VERSION}.dist-info" + _read_bounded_regular(dist_info / "METADATA") + _read_bounded_regular(dist_info / "RECORD") + try: + os.lstat(dist_info / "entry_points.txt") + except FileNotFoundError: + pass + except OSError: + _fail() + else: + _fail() + distribution = importlib.metadata.PathDistribution(dist_info) + try: + name = distribution.metadata["Name"] + version = distribution.metadata["Version"] + except KeyError: + _fail() + if name != _DISTRIBUTION_NAME or version != _DISTRIBUTION_VERSION: + _fail() + + records = _records(distribution) + launcher_record_path = PurePosixPath( + os.path.relpath(launcher_path, site_root) + ).as_posix() + launcher_record = records.get(launcher_record_path) + bootstrap_record = records.get(_BOOTSTRAP_FILE) + if launcher_record is None or bootstrap_record is None: + _fail() + + _, launcher_identity = _read_record_bound(launcher_path, launcher_record) + located_launcher = Path(str(distribution.locate_file(launcher_record))) + _, located_identity = _read_record_bound(located_launcher, launcher_record) + if launcher_identity != located_identity: + _fail() + + bootstrap_path = site_root / _BOOTSTRAP_FILE + payload, bootstrap_identity = _read_record_bound( + bootstrap_path, + bootstrap_record, + ) + located_bootstrap = Path(str(distribution.locate_file(bootstrap_record))) + _, located_bootstrap_identity = _read_record_bound( + located_bootstrap, + bootstrap_record, + ) + if bootstrap_identity != located_bootstrap_identity: + _fail() + + sys.path.append(str(site_root)) + return payload, bootstrap_path + + +def _execute_bootstrap(payload: bytes, bootstrap_path: Path) -> int: + forbidden = tuple( + name + for name in sys.modules + if name == "netveil_bootstrap" + or name == "netveil" + or name.startswith("netveil.") + ) + if forbidden: + _fail() + + module = ModuleType("netveil_bootstrap") + module.__file__ = str(bootstrap_path) + module.__package__ = "" + sys.modules[module.__name__] = module + try: + code = compile( + payload, + str(bootstrap_path), + "exec", + dont_inherit=True, + ) + exec(code, module.__dict__) # noqa: S102 - executes RECORD-bound bytes. + main = module.main + result = main(sys.argv[1:]) + except BaseException: + sys.modules.pop(module.__name__, None) + raise + if type(result) is not int: + _fail() + return result + + +def _write_failure(code: str, exit_code: int) -> int: + payload = f"netveil-audit: {code}\n" + offset = 0 + try: + while offset < len(payload): + written = sys.stderr.write(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return _OUTPUT_FAILURE_EXIT + offset += written + sys.stderr.flush() + except OSError: + return _OUTPUT_FAILURE_EXIT + return exit_code + + +def main() -> int: + try: + if not _startup_is_isolated(): + _fail() + launcher_path = Path(__file__).absolute() + payload, bootstrap_path = _verified_bootstrap(launcher_path) + return _execute_bootstrap(payload, bootstrap_path) + except _LaunchFailure: + return _write_failure("artifact_unverified", _ARTIFACT_FAILURE_EXIT) + except KeyboardInterrupt: + return _write_failure("interrupted", _INTERNAL_FAILURE_EXIT) + except Exception: # noqa: BLE001 - never expose launcher tracebacks. + return _write_failure("internal_error", _INTERNAL_FAILURE_EXIT) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/netveil/cli.py b/src/netveil/cli.py new file mode 100644 index 0000000..33a321c --- /dev/null +++ b/src/netveil/cli.py @@ -0,0 +1,361 @@ +"""Fail-closed command-line boundary for installed Netveil artifacts.""" + +from __future__ import annotations + +import argparse +import hmac +import os +import stat +import sys +from collections.abc import Sequence +from dataclasses import dataclass +from enum import IntEnum +from pathlib import Path +from typing import Any, BinaryIO, Final, NoReturn, TextIO + +from netveil.parser import MAX_INPUT_BYTES, EndpointParseError, parse_corpus +from netveil.privacy import ( + MIN_PSEUDONYMIZATION_KEY_BYTES, + build_privacy_receipt, +) + +_DISTRIBUTION_NAME: Final = "netveil-audit" +_DISTRIBUTION_VERSION: Final = "0.3.0" +_MAX_KEY_BYTES: Final = 4_096 + + +class CliExitCode(IntEnum): + """Stable process exit codes for automation.""" + + SUCCESS = 0 + USAGE = 2 + ARTIFACT_UNVERIFIED = 10 + CORPUS_UNAVAILABLE = 11 + KEY_UNAVAILABLE = 12 + CORPUS_REJECTED = 13 + KEY_REJECTED = 14 + OUTPUT_FAILED = 15 + INTERNAL_ERROR = 70 + + +class _CliFailure(Exception): + def __init__(self, code: str, exit_code: CliExitCode) -> None: + super().__init__(code) + self.code = code + self.exit_code = exit_code + + +class _CliCompletion(Exception): + def __init__(self, exit_code: CliExitCode) -> None: + super().__init__(int(exit_code)) + self.exit_code = exit_code + + +@dataclass(frozen=True, slots=True) +class _ReadFile: + payload: bytes + device: int + inode: int + + +class _SafeArgumentParser(argparse.ArgumentParser): + def error(self, message: str) -> NoReturn: + del message + raise _CliFailure("usage_error", CliExitCode.USAGE) + + def _print_message( + self, + message: str | None, + file: Any | None = None, + ) -> None: + if message and not _write_text(file or sys.stderr, message): + raise _CliFailure("output_failed", CliExitCode.OUTPUT_FAILED) + + def exit(self, status: int = 0, message: str | None = None) -> NoReturn: + if message is not None: + self._print_message( + message, + sys.stdout if status == 0 else sys.stderr, + ) + if status == 0: + raise _CliCompletion(CliExitCode.SUCCESS) + raise _CliFailure("usage_error", CliExitCode.USAGE) + + +def _fail(code: str, exit_code: CliExitCode) -> NoReturn: + raise _CliFailure(code, exit_code) + + +def _open_flags() -> int: + required_flags = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if os.name != "posix" or any(not hasattr(os, name) for name in required_flags): + _fail("platform_unsupported", CliExitCode.INTERNAL_ERROR) + return os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + + +def _file_identity(status: os.stat_result) -> tuple[int, ...]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_uid, + status.st_gid, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def _read_open_file( + descriptor: int, + *, + maximum_bytes: int, +) -> tuple[bytes | None, os.stat_result | None, os.stat_result | None]: + before: os.stat_result | None = None + after: os.stat_result | None = None + payload: bytes | None = None + operation_failed = False + try: + before = os.fstat(descriptor) + if stat.S_ISREG(before.st_mode) and before.st_size <= maximum_bytes: + chunks: list[bytes] = [] + observed = 0 + while observed <= maximum_bytes: + chunk = os.read( + descriptor, + min(65_536, maximum_bytes + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + operation_failed = True + if operation_failed: + return None, None, None + return payload, before, after + + +def _read_bounded_file( + path: Path, + *, + maximum_bytes: int, + failure_code: str, + failure_exit: CliExitCode, + key_policy: bool, +) -> _ReadFile: + descriptor: int | None = None + open_failed = False + try: + descriptor = os.open(path, _open_flags()) + except OSError: + open_failed = True + if open_failed or descriptor is None: + _fail(failure_code, failure_exit) + + payload: bytes | None = None + before: os.stat_result | None = None + after: os.stat_result | None = None + close_failed = False + try: + payload, before, after = _read_open_file( + descriptor, + maximum_bytes=maximum_bytes, + ) + finally: + try: + os.close(descriptor) + except OSError: + close_failed = True + + if ( + close_failed + or payload is None + or before is None + or after is None + or not stat.S_ISREG(before.st_mode) + or len(payload) > maximum_bytes + or len(payload) != after.st_size + or _file_identity(before) != _file_identity(after) + ): + _fail(failure_code, failure_exit) + + if key_policy and ( + len(payload) < MIN_PSEUDONYMIZATION_KEY_BYTES + or before.st_uid != os.geteuid() + or before.st_nlink != 1 + or not before.st_mode & stat.S_IRUSR + or before.st_mode + & ( + stat.S_IRWXG + | stat.S_IRWXO + | stat.S_IXUSR + | stat.S_ISUID + | stat.S_ISGID + | stat.S_ISVTX + ) + ): + _fail("key_rejected", CliExitCode.KEY_REJECTED) + return _ReadFile( + payload=payload, + device=before.st_dev, + inode=before.st_ino, + ) + + +def _parser(version: str) -> _SafeArgumentParser: + parser = _SafeArgumentParser( + prog="netveil-audit", + description="Create an offline pseudonymized endpoint-corpus receipt.", + allow_abbrev=False, + ) + parser.add_argument( + "--version", + action="store_true", + help="print the verified installed distribution version", + ) + subparsers = parser.add_subparsers(dest="command") + receipt = subparsers.add_parser( + "receipt", + help="write one canonical public receipt to standard output", + allow_abbrev=False, + ) + receipt.add_argument("corpus", type=Path, metavar="CORPUS") + receipt.add_argument( + "--key-file", + required=True, + type=Path, + metavar="KEY_FILE", + ) + parser.set_defaults(distribution_version=version) + return parser + + +def _write_text(stream: TextIO, payload: str) -> bool: + offset = 0 + try: + while offset < len(payload): + written = stream.write(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return False + offset += written + stream.flush() + except OSError: + return False + return True + + +def _write_binary(stream: BinaryIO, payload: bytes) -> bool: + offset = 0 + try: + while offset < len(payload): + written = stream.write(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return False + offset += written + stream.flush() + except (BrokenPipeError, OSError): + return False + return True + + +def _emit_failure(failure: _CliFailure) -> int: + if not _write_text(sys.stderr, f"netveil-audit: {failure.code}\n"): + return int(CliExitCode.OUTPUT_FAILED) + return int(failure.exit_code) + + +def _write_receipt(payload: bytes) -> None: + if not _write_binary(sys.stdout.buffer, payload + b"\n"): + _fail("output_failed", CliExitCode.OUTPUT_FAILED) + + +def _run_receipt(corpus_path: Path, key_path: Path) -> None: + corpus_file = _read_bounded_file( + corpus_path, + maximum_bytes=MAX_INPUT_BYTES, + failure_code="corpus_unavailable", + failure_exit=CliExitCode.CORPUS_UNAVAILABLE, + key_policy=False, + ) + parse_failure: str | None = None + try: + parse_corpus(corpus_file.payload) + except EndpointParseError as error: + location = "" if error.line_number is None else f":line={error.line_number}" + parse_failure = f"corpus_rejected:{error.code.value}{location}" + if parse_failure is not None: + _fail( + parse_failure, + CliExitCode.CORPUS_REJECTED, + ) + + key_file = _read_bounded_file( + key_path, + maximum_bytes=_MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + if (corpus_file.device, corpus_file.inode) == (key_file.device, key_file.inode) or ( + len(corpus_file.payload) == len(key_file.payload) + and hmac.compare_digest(corpus_file.payload, key_file.payload) + ): + _fail("key_rejected", CliExitCode.KEY_REJECTED) + receipt = build_privacy_receipt( + corpus_file.payload, + pseudonymization_key=key_file.payload, + ) + _write_receipt(receipt.canonical_json_bytes()) + + +def _main_verified( + argv: Sequence[str] | None = None, + *, + verified_distribution_version: str | None = None, +) -> int: + """Run only after the external bootstrap has verified source bytes.""" + + try: + if verified_distribution_version != _DISTRIBUTION_VERSION: + _fail("artifact_unverified", CliExitCode.ARTIFACT_UNVERIFIED) + raw_arguments = tuple(sys.argv[1:] if argv is None else argv) + if "--version" in raw_arguments and raw_arguments != ("--version",): + _fail("usage_error", CliExitCode.USAGE) + key_option_count = sum( + argument == "--key-file" or argument.startswith("--key-file=") + for argument in raw_arguments + ) + if key_option_count > 1 or "--key-file=" in raw_arguments: + _fail("usage_error", CliExitCode.USAGE) + arguments = _parser(verified_distribution_version).parse_args(raw_arguments) + if arguments.version: + if not _write_text( + sys.stdout, + f"{_DISTRIBUTION_NAME} {verified_distribution_version}\n", + ): + _fail("output_failed", CliExitCode.OUTPUT_FAILED) + return int(CliExitCode.SUCCESS) + if arguments.command != "receipt": + _fail("usage_error", CliExitCode.USAGE) + _run_receipt(arguments.corpus, arguments.key_file) + except _CliCompletion as completion: + return int(completion.exit_code) + except _CliFailure as failure: + return _emit_failure(failure) + except KeyboardInterrupt: + return _emit_failure(_CliFailure("interrupted", CliExitCode.INTERNAL_ERROR)) + except Exception: # noqa: BLE001 - CLI must never render sensitive tracebacks. + return _emit_failure(_CliFailure("internal_error", CliExitCode.INTERNAL_ERROR)) + return int(CliExitCode.SUCCESS) diff --git a/src/netveil_bootstrap.py b/src/netveil_bootstrap.py new file mode 100644 index 0000000..e9f0b3b --- /dev/null +++ b/src/netveil_bootstrap.py @@ -0,0 +1,540 @@ +"""Stdlib-only integrity bootstrap for the installed Netveil command.""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib +import importlib.abc +import importlib.util +import os +import stat +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from importlib import metadata +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Final, NoReturn, TextIO + +_DISTRIBUTION_NAME: Final = "netveil-audit" +_DISTRIBUTION_VERSION: Final = "0.3.0" +_BOOTSTRAP_FILE: Final = "netveil_bootstrap.py" +_MAX_ARTIFACT_FILE_BYTES: Final = 1_048_576 +_ARTIFACT_FAILURE_EXIT: Final = 10 +_OUTPUT_FAILURE_EXIT: Final = 15 +_INTERNAL_FAILURE_EXIT: Final = 70 +_SOURCE_MODULES: Final = ( + ("netveil", "__init__.py", True), + ("netveil.cli", "cli.py", False), + ("netveil.model", "model.py", False), + ("netveil.parser", "parser.py", False), + ("netveil.privacy", "privacy.py", False), +) +_PACKAGE_FILES: Final = frozenset( + {filename for _, filename, _ in _SOURCE_MODULES} | {"py.typed"} +) +_METADATA_FILES: Final = ( + "METADATA", + "WHEEL", + "top_level.txt", +) + + +class _BootstrapFailure(Exception): + def __init__(self, code: str, exit_code: int) -> None: + super().__init__(code) + self.code = code + self.exit_code = exit_code + + +@dataclass(frozen=True, slots=True) +class _PinnedBytes: + payload: bytes + identity: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class _VerifiedArtifact: + version: str + package_root: Path + sources: tuple[tuple[str, str, bytes, bool], ...] + + +def _fail(code: str = "artifact_unverified") -> NoReturn: + raise _BootstrapFailure(code, _ARTIFACT_FAILURE_EXIT) + + +def _identity(status: os.stat_result) -> tuple[int, ...]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_uid, + status.st_gid, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def _file_flags() -> int: + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if os.name != "posix" or any(not hasattr(os, name) for name in required): + _fail("platform_unsupported") + return os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + + +def _directory_flags() -> int: + if not hasattr(os, "O_DIRECTORY"): + _fail("platform_unsupported") + return _file_flags() | os.O_DIRECTORY + + +def _close(descriptor: int) -> None: + try: + os.close(descriptor) + except OSError: + _fail() + + +def _read_descriptor( + descriptor: int, + *, + expected_size: int, +) -> _PinnedBytes: + if ( + type(expected_size) is not int + or expected_size < 0 + or expected_size > _MAX_ARTIFACT_FILE_BYTES + ): + _fail() + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_size != expected_size: + _fail() + chunks: list[bytes] = [] + observed = 0 + while observed <= expected_size: + chunk = os.read( + descriptor, + min(65_536, expected_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail() + if ( + len(payload) != expected_size + or _identity(before) != _identity(after) + or after.st_size != len(payload) + ): + _fail() + return _PinnedBytes(payload=payload, identity=_identity(after)) + + +def _record_contract(record: metadata.PackagePath) -> tuple[int, str]: + file_hash = record.hash + file_size = record.size + if ( + file_hash is None + or file_hash.mode != "sha256" + or type(file_size) is not int + or file_size < 0 + or file_size > _MAX_ARTIFACT_FILE_BYTES + ): + _fail() + return file_size, file_hash.value + + +def _sha256_record_value(payload: bytes) -> str: + digest = hashlib.sha256(payload).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def _read_path( + path: Path, + *, + record: metadata.PackagePath, +) -> _PinnedBytes: + expected_size, expected_hash = _record_contract(record) + descriptor = -1 + try: + descriptor = os.open(path, _file_flags()) + pinned = _read_descriptor(descriptor, expected_size=expected_size) + except OSError: + _fail() + finally: + if descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + _close(descriptor) + if _sha256_record_value(pinned.payload) != expected_hash: + _fail() + return pinned + + +def _read_package_file( + root_descriptor: int, + *, + name: str, + record: metadata.PackagePath, + distribution: metadata.Distribution, +) -> bytes: + expected_size, expected_hash = _record_contract(record) + descriptor = -1 + located_descriptor = -1 + try: + descriptor = os.open( + name, + _file_flags(), + dir_fd=root_descriptor, + ) + pinned = _read_descriptor(descriptor, expected_size=expected_size) + located = Path(str(distribution.locate_file(record))) + located_descriptor = os.open(located, _file_flags()) + located_status = os.fstat(located_descriptor) + current_status = os.stat( + name, + dir_fd=root_descriptor, + follow_symlinks=False, + ) + except OSError: + _fail() + finally: + if located_descriptor >= 0: + _close(located_descriptor) + if descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + _close(descriptor) + if ( + _identity(located_status) != pinned.identity + or _identity(current_status) != pinned.identity + or _sha256_record_value(pinned.payload) != expected_hash + ): + _fail() + return pinned.payload + + +def _cache_names() -> frozenset[str]: + cache_tag = sys.implementation.cache_tag + if type(cache_tag) is not str or not cache_tag: + _fail() + return frozenset( + f"{Path(filename).stem}.{cache_tag}.pyc" for _, filename, _ in _SOURCE_MODULES + ) + + +def _verify_cache_directory(root_descriptor: int) -> None: + descriptor = -1 + try: + descriptor = os.open( + "__pycache__", + _directory_flags(), + dir_fd=root_descriptor, + ) + before = os.fstat(descriptor) + if not stat.S_ISDIR(before.st_mode): + _fail() + names = os.listdir(descriptor) + if len(names) != len(set(names)) or not set(names).issubset(_cache_names()): + _fail() + for name in names: + status = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if not stat.S_ISREG(status.st_mode): + _fail() + after = os.fstat(descriptor) + except OSError: + _fail() + finally: + if descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + _close(descriptor) + if _identity(before) != _identity(after): + _fail() + + +def _scan_package_directory(root_descriptor: int) -> frozenset[str]: + try: + names = os.listdir(root_descriptor) + except OSError: + _fail() + if len(names) != len(set(names)): + _fail() + observed = frozenset(names) + allowed = _PACKAGE_FILES | {"__pycache__"} + if not _PACKAGE_FILES.issubset(observed) or not observed.issubset(allowed): + _fail() + if "__pycache__" in observed: + _verify_cache_directory(root_descriptor) + return observed + + +def _collect_records( + distribution: metadata.Distribution, +) -> tuple[ + dict[str, metadata.PackagePath], + metadata.PackagePath, + dict[str, metadata.PackagePath], +]: + files = distribution.files + if files is None: + _fail() + records: dict[str, metadata.PackagePath] = {} + for record in files: + raw_path = str(record) + if raw_path in records: + _fail() + records[raw_path] = record + + package_records: dict[str, metadata.PackagePath] = {} + allowed_cache_paths = {f"netveil/__pycache__/{name}" for name in _cache_names()} + for raw_path, record in records.items(): + path = PurePosixPath(raw_path) + if not path.parts or path.parts[0] != "netveil": + continue + if len(path.parts) == 2 and path.name in _PACKAGE_FILES: + if path.name in package_records: + _fail() + package_records[path.name] = record + elif raw_path not in allowed_cache_paths: + _fail() + if set(package_records) != _PACKAGE_FILES: + _fail() + + bootstrap_record = records.get(_BOOTSTRAP_FILE) + if bootstrap_record is None: + _fail() + + metadata_root = f"netveil_audit-{_DISTRIBUTION_VERSION}.dist-info" + metadata_records: dict[str, metadata.PackagePath] = {} + for name in _METADATA_FILES: + metadata_record = records.get(f"{metadata_root}/{name}") + if metadata_record is None: + _fail() + metadata_records[name] = metadata_record + return package_records, bootstrap_record, metadata_records + + +def _verify_distribution_metadata( + distribution: metadata.Distribution, + records: Mapping[str, metadata.PackagePath], +) -> None: + try: + project_name = distribution.metadata["Name"] + project_version = distribution.metadata["Version"] + except KeyError: + _fail() + if ( + distribution.version != _DISTRIBUTION_VERSION + or project_name != _DISTRIBUTION_NAME + or project_version != _DISTRIBUTION_VERSION + ): + _fail() + if list(distribution.entry_points): + _fail() + + payloads = { + name: _read_path( + Path(str(distribution.locate_file(record))), + record=record, + ).payload + for name, record in records.items() + } + if payloads["top_level.txt"] != b"netveil\nnetveil_bootstrap\n": + _fail() + + +def _verify_installed_artifact() -> _VerifiedArtifact: + """Bind installed source bytes before any ``netveil`` package import.""" + + if any(name == "netveil" or name.startswith("netveil.") for name in sys.modules): + _fail() + try: + distribution = metadata.distribution(_DISTRIBUTION_NAME) + except metadata.PackageNotFoundError: + _fail() + + package_records, bootstrap_record, metadata_records = _collect_records(distribution) + _verify_distribution_metadata(distribution, metadata_records) + + bootstrap_path = Path(__file__) + bootstrap_pinned = _read_path(bootstrap_path, record=bootstrap_record) + try: + recorded_bootstrap = Path(str(distribution.locate_file(bootstrap_record))) + recorded_pinned = _read_path( + recorded_bootstrap, + record=bootstrap_record, + ) + except OSError: + _fail() + if bootstrap_pinned.identity != recorded_pinned.identity: + _fail() + + package_root = bootstrap_path.parent / "netveil" + root_descriptor = -1 + recorded_root_descriptor = -1 + try: + root_descriptor = os.open(package_root, _directory_flags()) + root_before = os.fstat(root_descriptor) + if not stat.S_ISDIR(root_before.st_mode): + _fail() + recorded_root = Path(str(distribution.locate_file("netveil"))) + recorded_root_descriptor = os.open(recorded_root, _directory_flags()) + recorded_root_status = os.fstat(recorded_root_descriptor) + if _identity(root_before) != _identity(recorded_root_status): + _fail() + observed_before = _scan_package_directory(root_descriptor) + payloads = { + name: _read_package_file( + root_descriptor, + name=name, + record=package_records[name], + distribution=distribution, + ) + for name in sorted(_PACKAGE_FILES) + } + observed_after = _scan_package_directory(root_descriptor) + root_after = os.fstat(root_descriptor) + except OSError: + _fail() + finally: + if recorded_root_descriptor >= 0: + _close(recorded_root_descriptor) + if root_descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + _close(root_descriptor) + if observed_before != observed_after or _identity(root_before) != _identity( + root_after + ): + _fail() + + sources = tuple( + (module, filename, payloads[filename], is_package) + for module, filename, is_package in _SOURCE_MODULES + ) + return _VerifiedArtifact( + version=distribution.version, + package_root=package_root, + sources=sources, + ) + + +class _VerifiedSourceLoader(importlib.abc.Loader): + def __init__(self, *, source: bytes, origin: str) -> None: + self._source = source + self._origin = origin + + def create_module(self, spec: importlib.machinery.ModuleSpec) -> ModuleType | None: + del spec + return None + + def exec_module(self, module: ModuleType) -> None: + code = compile( + self._source, + self._origin, + "exec", + dont_inherit=True, + ) + exec(code, module.__dict__) # noqa: S102 - executes only verified bytes. + + +class _VerifiedSourceFinder(importlib.abc.MetaPathFinder): + def __init__(self, artifact: _VerifiedArtifact) -> None: + self._sources = { + module: ( + artifact.package_root / filename, + payload, + is_package, + ) + for module, filename, payload, is_package in artifact.sources + } + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None, + target: ModuleType | None = None, + ) -> importlib.machinery.ModuleSpec | None: + del path, target + if fullname != "netveil" and not fullname.startswith("netveil."): + return None + source = self._sources.get(fullname) + if source is None: + raise ModuleNotFoundError("verified Netveil module is not allowed") + origin, payload, is_package = source + loader = _VerifiedSourceLoader(source=payload, origin=str(origin)) + spec = importlib.util.spec_from_loader( + fullname, + loader, + origin=str(origin), + is_package=is_package, + ) + if spec is None: + raise ModuleNotFoundError("verified Netveil module is unavailable") + return spec + + +def _load_verified_cli(artifact: _VerifiedArtifact) -> ModuleType: + if any(name == "netveil" or name.startswith("netveil.") for name in sys.modules): + _fail() + finder = _VerifiedSourceFinder(artifact) + sys.meta_path.insert(0, finder) + try: + cli = importlib.import_module("netveil.cli") + except BaseException: # Cleanup must cover import-time exits. + for name in tuple(sys.modules): + if name == "netveil" or name.startswith("netveil."): + sys.modules.pop(name, None) + if finder in sys.meta_path: + sys.meta_path.remove(finder) + raise + return cli + + +def _write_text(stream: TextIO, payload: str) -> bool: + offset = 0 + try: + while offset < len(payload): + written = stream.write(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return False + offset += written + stream.flush() + except OSError: + return False + return True + + +def _emit_failure(code: str, exit_code: int) -> int: + if not _write_text(sys.stderr, f"netveil-audit: {code}\n"): + return _OUTPUT_FAILURE_EXIT + return exit_code + + +def main(argv: Sequence[str] | None = None) -> int: + """Verify, source-load, and execute the installed command.""" + + try: + artifact = _verify_installed_artifact() + cli = _load_verified_cli(artifact) + run = cli._main_verified + result = run( + argv, + verified_distribution_version=artifact.version, + ) + if type(result) is not int: + raise TypeError + return result + except _BootstrapFailure as failure: + return _emit_failure(failure.code, failure.exit_code) + except KeyboardInterrupt: + return _emit_failure("interrupted", _INTERNAL_FAILURE_EXIT) + except SystemExit: + return _emit_failure("internal_error", _INTERNAL_FAILURE_EXIT) + except Exception: # noqa: BLE001 - never expose bootstrap tracebacks. + return _emit_failure("internal_error", _INTERNAL_FAILURE_EXIT) diff --git a/tests/fixtures/launcher_bootstrap_fixture.py b/tests/fixtures/launcher_bootstrap_fixture.py new file mode 100644 index 0000000..21e1dad --- /dev/null +++ b/tests/fixtures/launcher_bootstrap_fixture.py @@ -0,0 +1,7 @@ +"""Minimal source-loaded bootstrap fixture for launcher boundary tests.""" + + +def main(arguments: list[str]) -> int: + if arguments[0] == "explode": + raise RuntimeError("fixture failure") + return {"invalid": True, "safe": 23}[arguments[0]] diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py new file mode 100644 index 0000000..d7fa81a --- /dev/null +++ b/tests/test_bootstrap.py @@ -0,0 +1,1040 @@ +from __future__ import annotations + +import base64 +import contextlib +import hashlib +import importlib +import io +import os +import py_compile +import stat +import subprocess +import sys +import tempfile +import unittest +from collections.abc import Callable, Iterator +from dataclasses import dataclass, replace +from importlib import metadata +from pathlib import Path +from types import ModuleType +from typing import cast +from unittest.mock import patch + +import netveil_bootstrap as bootstrap + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_ROOT = ROOT / "src" + + +@dataclass(slots=True) +class _FakeHash: + mode: str + value: str + + +@dataclass(slots=True) +class _FakeRecord: + path: str + hash: _FakeHash | None + size: int | None + + def __str__(self) -> str: + return self.path + + +@dataclass(slots=True) +class _FakeEntryPoint: + group: str + name: str + value: str + + +@dataclass +class _FakeDistribution: + root: Path + version: str + files: list[_FakeRecord] | None + metadata: dict[str, str] + entry_points: list[_FakeEntryPoint] + + def locate_file(self, path: object) -> Path: + return self.root / str(path) + + +def _hash(payload: bytes) -> str: + digest = hashlib.sha256(payload).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def _record(path: str, payload: bytes) -> _FakeRecord: + return _FakeRecord(path, _FakeHash("sha256", _hash(payload)), len(payload)) + + +def _write(root: Path, relative: str, payload: bytes) -> _FakeRecord: + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(payload) + return _record(relative, payload) + + +def _create_artifact(root: Path) -> _FakeDistribution: + records: list[_FakeRecord] = [] + records.append( + _write( + root, + bootstrap._BOOTSTRAP_FILE, + (SOURCE_ROOT / bootstrap._BOOTSTRAP_FILE).read_bytes(), + ) + ) + for name in sorted(bootstrap._PACKAGE_FILES): + records.append( + _write( + root, + f"netveil/{name}", + (SOURCE_ROOT / "netveil" / name).read_bytes(), + ) + ) + + metadata_root = f"netveil_audit-{bootstrap._DISTRIBUTION_VERSION}.dist-info" + metadata_payloads = { + "METADATA": (b"Metadata-Version: 2.4\nName: netveil-audit\nVersion: 0.3.0\n"), + "WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + "top_level.txt": b"netveil\nnetveil_bootstrap\n", + } + for name, payload in metadata_payloads.items(): + records.append(_write(root, f"{metadata_root}/{name}", payload)) + + return _FakeDistribution( + root=root, + version=bootstrap._DISTRIBUTION_VERSION, + files=records, + metadata={ + "Name": bootstrap._DISTRIBUTION_NAME, + "Version": bootstrap._DISTRIBUTION_VERSION, + }, + entry_points=[], + ) + + +@contextlib.contextmanager +def _isolated_netveil_modules() -> Iterator[None]: + saved_modules = { + name: module + for name, module in sys.modules.items() + if name == "netveil" or name.startswith("netveil.") + } + saved_meta_path = list(sys.meta_path) + for name in saved_modules: + sys.modules.pop(name, None) + try: + yield + finally: + for name in tuple(sys.modules): + if name == "netveil" or name.startswith("netveil."): + sys.modules.pop(name, None) + sys.modules.update(saved_modules) + sys.meta_path[:] = saved_meta_path + + +def _verify( + root: Path, + distribution: _FakeDistribution, +) -> bootstrap._VerifiedArtifact: + with ( + _isolated_netveil_modules(), + patch.object(bootstrap, "__file__", str(root / bootstrap._BOOTSTRAP_FILE)), + patch.object( + metadata, + "distribution", + return_value=cast(metadata.Distribution, distribution), + ), + ): + return bootstrap._verify_installed_artifact() + + +class BootstrapArtifactTests(unittest.TestCase): + def test_importing_bootstrap_does_not_import_package(self) -> None: + command = ( + "import sys;" + f"sys.path.insert(0,{str(SOURCE_ROOT)!r});" + "import netveil_bootstrap;" + "print(any(n == 'netveil' or n.startswith('netveil.') " + "for n in sys.modules))" + ) + completed = subprocess.run( + (sys.executable, "-I", "-B", "-c", command), + check=True, + capture_output=True, + text=True, + env={ + "HOME": "/nonexistent", + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": os.environ.get("PATH", ""), + "PYTHONHASHSEED": "0", + "TZ": "UTC", + }, + ) + self.assertEqual(completed.stdout, "False\n") + self.assertEqual(completed.stderr, "") + + def test_exact_artifact_returns_pinned_source_bytes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + distribution = _create_artifact(root) + artifact = _verify(root, distribution) + self.assertEqual(artifact.version, bootstrap._DISTRIBUTION_VERSION) + self.assertEqual( + {module for module, _, _, _ in artifact.sources}, + {module for module, _, _ in bootstrap._SOURCE_MODULES}, + ) + self.assertTrue(all(payload for _, _, payload, _ in artifact.sources)) + + def test_loaded_package_and_missing_distribution_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _create_artifact(root) + with ( + patch.dict(sys.modules, {"netveil": ModuleType("netveil")}), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_installed_artifact() + + with ( + _isolated_netveil_modules(), + patch.object( + metadata, + "distribution", + side_effect=metadata.PackageNotFoundError("netveil-audit"), + ), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_installed_artifact() + + def test_metadata_identity_and_entrypoint_are_exact(self) -> None: + mutations = ("version", "name", "metadata-version", "entrypoint") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + if mutation == "version": + distribution.version = "9.9.9" + elif mutation == "name": + distribution.metadata["Name"] = "other" + elif mutation == "metadata-version": + distribution.metadata["Version"] = "9.9.9" + else: + distribution.entry_points.append( + _FakeEntryPoint( + group="console_scripts", + name="netveil-audit", + value="netveil.cli:entrypoint", + ) + ) + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_missing_duplicate_nested_and_extra_package_records_fail(self) -> None: + mutations = ("missing", "duplicate", "nested", "extra-disk") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + assert distribution.files is not None + if mutation == "missing": + distribution.files = [ + record + for record in distribution.files + if str(record) != "netveil/parser.py" + ] + elif mutation == "duplicate": + distribution.files.append(distribution.files[0]) + elif mutation == "nested": + distribution.files.append( + _FakeRecord( + "netveil/plugins/evil.py", + _FakeHash("sha256", "x"), + 1, + ) + ) + else: + (root / "netveil" / "evil.py").write_text("raise SystemExit\n") + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_hash_size_type_and_location_drift_fail(self) -> None: + mutations = ( + "hash", + "algorithm", + "size", + "size-type", + "oversized", + "symlink", + ) + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + assert distribution.files is not None + record = next( + item + for item in distribution.files + if str(item) == "netveil/parser.py" + ) + if mutation == "hash": + assert record.hash is not None + record.hash.value = "wrong" + elif mutation == "algorithm": + assert record.hash is not None + record.hash.mode = "sha512" + elif mutation == "size": + assert record.size is not None + record.size += 1 + elif mutation == "size-type": + record.size = True + elif mutation == "oversized": + record.size = bootstrap._MAX_ARTIFACT_FILE_BYTES + 1 + else: + target = root / "netveil" / "parser.py" + target.unlink() + target.symlink_to(root / "netveil" / "model.py") + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_metadata_payloads_and_top_levels_are_record_bound(self) -> None: + mutations = ("payload", "top-level", "entry-point", "missing-record") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + assert distribution.files is not None + metadata_root = ( + f"netveil_audit-{bootstrap._DISTRIBUTION_VERSION}.dist-info" + ) + if mutation == "payload": + (root / metadata_root / "WHEEL").write_bytes(b"tampered\n") + elif mutation == "top-level": + path = root / metadata_root / "top_level.txt" + payload = b"netveil_bootstrap\nnetveil\n" + path.write_bytes(payload) + record = next( + item + for item in distribution.files + if str(item).endswith("/top_level.txt") + ) + record.hash = _FakeHash("sha256", _hash(payload)) + record.size = len(payload) + elif mutation == "entry-point": + distribution.entry_points.append( + _FakeEntryPoint( + group="console_scripts", + name="other", + value="other:main", + ) + ) + else: + distribution.files = [ + item + for item in distribution.files + if not str(item).endswith("/METADATA") + ] + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_cache_is_inert_but_its_shape_is_bounded(self) -> None: + mutations = ("valid", "unknown", "symlink", "directory") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + package_root = root / "netveil" + cache = package_root / "__pycache__" + cache.mkdir() + known = next(iter(sorted(bootstrap._cache_names()))) + target = cache / known + if mutation == "valid": + target.write_bytes(b"untrusted and deliberately unread") + assert distribution.files is not None + distribution.files.append( + _FakeRecord( + f"netveil/__pycache__/{known}", + None, + None, + ) + ) + artifact = _verify(root, distribution) + self.assertEqual( + artifact.version, + bootstrap._DISTRIBUTION_VERSION, + ) + continue + if mutation == "unknown": + target = cache / "evil.pyc" + target.write_bytes(b"x") + elif mutation == "symlink": + target.symlink_to(package_root / "model.py") + else: + target.mkdir() + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + +class BootstrapSourceLoaderTests(unittest.TestCase): + def test_verified_bytes_ignore_disk_tamper_and_unchecked_hash_pyc(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + distribution = _create_artifact(root) + package_root = root / "netveil" + cache = package_root / "__pycache__" + cache.mkdir() + cache_name = f"cli.{sys.implementation.cache_tag}.pyc" + malicious_source = root / "malicious.py" + marker = root / "BYTECODE_EXECUTED" + malicious_source.write_text( + "from pathlib import Path\n" + f"Path({str(marker)!r}).write_text('executed')\n" + ) + py_compile.compile( + str(malicious_source), + cfile=str(cache / cache_name), + doraise=True, + invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH, + ) + assert distribution.files is not None + distribution.files.append( + _FakeRecord( + f"netveil/__pycache__/{cache_name}", + None, + None, + ) + ) + + with ( + _isolated_netveil_modules(), + patch.object( + bootstrap, + "__file__", + str(root / bootstrap._BOOTSTRAP_FILE), + ), + patch.object( + metadata, + "distribution", + return_value=cast(metadata.Distribution, distribution), + ), + ): + artifact = bootstrap._verify_installed_artifact() + artifact = replace( + artifact, + package_root=SOURCE_ROOT / "netveil", + ) + (package_root / "cli.py").write_text( + "raise AssertionError('disk source executed')\n" + ) + cli = bootstrap._load_verified_cli(artifact) + self.assertEqual( + cli._DISTRIBUTION_VERSION, + bootstrap._DISTRIBUTION_VERSION, + ) + with self.assertRaises(ModuleNotFoundError): + importlib.import_module("netveil.evil") + self.assertFalse(marker.exists()) + + def test_finder_ignores_other_modules_and_rejects_unknown_netveil_module( + self, + ) -> None: + artifact = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=Path("/verified"), + sources=(("netveil", "__init__.py", b"", True),), + ) + finder = bootstrap._VerifiedSourceFinder(artifact) + self.assertIsNone(finder.find_spec("json", None)) + with self.assertRaises(ModuleNotFoundError): + finder.find_spec("netveil.evil", None) + spec = finder.find_spec("netveil", None) + self.assertIsNotNone(spec) + + +class BootstrapBoundaryTests(unittest.TestCase): + def test_descriptor_reader_rejects_growth_nonregular_and_read_failure( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "payload" + path.write_bytes(b"abc") + descriptor = os.open(path, os.O_RDONLY) + try: + current = os.fstat(descriptor) + changed = os.stat_result( + ( + current.st_mode, + current.st_ino, + current.st_dev, + current.st_nlink, + current.st_uid, + current.st_gid, + current.st_size + 1, + current.st_atime, + current.st_mtime, + current.st_ctime, + ) + ) + with ( + patch.object(os, "fstat", side_effect=(current, changed)), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_descriptor(descriptor, expected_size=3) + with ( + patch.object(os, "read", side_effect=OSError("PRIVATE")), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_descriptor(descriptor, expected_size=3) + finally: + os.close(descriptor) + + descriptor, writable = os.pipe() + try: + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._read_descriptor(descriptor, expected_size=0) + finally: + os.close(descriptor) + os.close(writable) + + def test_platform_and_record_contract_fail_closed(self) -> None: + with ( + patch.object(os, "name", "unsupported"), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._file_flags() + record = cast( + metadata.PackagePath, + _FakeRecord("x", None, None), + ) + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._record_contract(record) + + def test_directory_flag_close_and_invalid_size_failures(self) -> None: + directory_flag = os.O_DIRECTORY + del os.O_DIRECTORY + try: + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._directory_flags() + finally: + os.O_DIRECTORY = directory_flag # type: ignore[misc] + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "payload" + path.write_bytes(b"x") + descriptor = os.open(path, os.O_RDONLY) + try: + with ( + patch.object(os, "close", side_effect=OSError("PRIVATE")), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._close(descriptor) + finally: + os.close(descriptor) + + for invalid in (-1, True, bootstrap._MAX_ARTIFACT_FILE_BYTES + 1): + with ( + self.subTest(invalid=invalid), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_descriptor(-1, expected_size=invalid) + + def test_path_reader_rejects_missing_same_size_tamper_and_overread(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload = b"abc" + path = root / "payload" + path.write_bytes(payload) + record = cast(metadata.PackagePath, _record("payload", payload)) + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._read_path(root / "missing", record=record) + with ( + patch.object(os, "open", return_value=-1), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_path(path, record=record) + + wrong = cast( + metadata.PackagePath, + _FakeRecord( + "payload", + _FakeHash("sha256", _hash(b"xyz")), + len(payload), + ), + ) + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._read_path(path, record=wrong) + + descriptor = os.open(path, os.O_RDONLY) + try: + current = os.fstat(descriptor) + with ( + patch.object(os, "fstat", side_effect=(current, current)), + patch.object(os, "read", return_value=b"abcd"), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_descriptor(descriptor, expected_size=3) + finally: + os.close(descriptor) + + def test_cache_and_package_scan_low_level_failures(self) -> None: + implementation = sys.implementation + with ( + patch.object( + sys, + "implementation", + type("Implementation", (), {"cache_tag": None})(), + ), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._cache_names() + self.assertIs(sys.implementation, implementation) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cache = root / "__pycache__" + cache.mkdir() + root_descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY) + cache_descriptor = os.open(cache, os.O_RDONLY | os.O_DIRECTORY) + try: + cache_status = os.fstat(cache_descriptor) + regular_status = os.stat_result( + ( + stat.S_IFREG | 0o600, + cache_status.st_ino, + cache_status.st_dev, + cache_status.st_nlink, + cache_status.st_uid, + cache_status.st_gid, + cache_status.st_size, + cache_status.st_atime, + cache_status.st_mtime, + cache_status.st_ctime, + ) + ) + with ( + patch.object(os, "open", return_value=cache_descriptor), + patch.object(os, "fstat", return_value=regular_status), + patch.object(os, "close"), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_cache_directory(root_descriptor) + + with ( + patch.object(os, "open", side_effect=OSError("PRIVATE")), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_cache_directory(root_descriptor) + with ( + patch.object(os, "open", return_value=-1), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_cache_directory(root_descriptor) + + changed_status = os.stat_result( + ( + cache_status.st_mode, + cache_status.st_ino, + cache_status.st_dev, + cache_status.st_nlink, + cache_status.st_uid, + cache_status.st_gid, + cache_status.st_size, + cache_status.st_atime, + cache_status.st_mtime + 1, + cache_status.st_ctime, + ) + ) + with ( + patch.object(os, "open", return_value=cache_descriptor), + patch.object( + os, + "fstat", + side_effect=(cache_status, changed_status), + ), + patch.object(os, "listdir", return_value=[]), + patch.object(os, "close"), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_cache_directory(root_descriptor) + + with ( + patch.object(os, "listdir", side_effect=OSError("PRIVATE")), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._scan_package_directory(root_descriptor) + with ( + patch.object(os, "listdir", return_value=["x", "x"]), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._scan_package_directory(root_descriptor) + finally: + os.close(cache_descriptor) + os.close(root_descriptor) + + def test_record_collection_and_metadata_rare_failures(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + distribution = _create_artifact(root) + distribution.files = None + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._collect_records(cast(metadata.Distribution, distribution)) + + mutations = ("normalized-duplicate", "bootstrap", "metadata-key", "unicode") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + assert distribution.files is not None + if mutation == "normalized-duplicate": + source = next( + item + for item in distribution.files + if str(item) == "netveil/parser.py" + ) + distribution.files.append( + _FakeRecord( + "netveil/./parser.py", + source.hash, + source.size, + ) + ) + elif mutation == "bootstrap": + distribution.files = [ + item + for item in distribution.files + if str(item) != bootstrap._BOOTSTRAP_FILE + ] + elif mutation == "metadata-key": + distribution.metadata.pop("Name") + else: + metadata_root = ( + f"netveil_audit-{bootstrap._DISTRIBUTION_VERSION}.dist-info" + ) + path = root / metadata_root / "top_level.txt" + payload = b"\xffetveil\nnetveil_bootstrap\n" + path.write_bytes(payload) + record = next( + item + for item in distribution.files + if str(item).endswith("/top_level.txt") + ) + record.hash = _FakeHash("sha256", _hash(payload)) + record.size = len(payload) + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_artifact_location_directory_and_terminal_identity_failures(self) -> None: + mutations = ( + "bootstrap-locate", + "bootstrap-inode", + "root-kind", + "root-locate", + "root-inode", + "root-missing", + "terminal-drift", + ) + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + original_locate = distribution.locate_file + locate_override: Callable[[object], Path] | None = None + if mutation == "bootstrap-locate": + + def locate_bootstrap_failure( + path: object, + fallback: Callable[[object], Path] = original_locate, + ) -> Path: + if str(path) == bootstrap._BOOTSTRAP_FILE: + raise OSError("PRIVATE") + return fallback(path) + + locate_override = locate_bootstrap_failure + elif mutation == "bootstrap-inode": + copy = root / "bootstrap-copy.py" + copy.write_bytes((root / bootstrap._BOOTSTRAP_FILE).read_bytes()) + + def locate_bootstrap_copy( + path: object, + copy_path: Path = copy, + fallback: Callable[[object], Path] = original_locate, + ) -> Path: + if str(path) == bootstrap._BOOTSTRAP_FILE: + return copy_path + return fallback(path) + + locate_override = locate_bootstrap_copy + elif mutation == "root-kind": + kind = stat.S_ISDIR + package_mode = (root / "netveil").stat().st_mode + + def is_directory( + mode: int, + *, + expected_mode: int = package_mode, + fallback: Callable[[int], bool] = kind, + ) -> bool: + if mode == expected_mode: + return False + return fallback(mode) + + with ( + patch.object(stat, "S_ISDIR", side_effect=is_directory), + self.assertRaises(bootstrap._BootstrapFailure), + ): + _verify(root, distribution) + continue + elif mutation == "root-locate": + + def locate_root_failure( + path: object, + fallback: Callable[[object], Path] = original_locate, + ) -> Path: + if str(path) == "netveil": + raise OSError("PRIVATE") + return fallback(path) + + locate_override = locate_root_failure + elif mutation == "root-inode": + other = root / "other-netveil" + other.mkdir() + + def locate_other_root( + path: object, + other_root: Path = other, + fallback: Callable[[object], Path] = original_locate, + ) -> Path: + if str(path) == "netveil": + return other_root + return fallback(path) + + locate_override = locate_other_root + elif mutation == "root-missing": + package_root = root / "netveil" + moved_root = root / "moved-netveil" + package_root.rename(moved_root) + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + continue + else: + original_scan = bootstrap._scan_package_directory + calls = 0 + + def drift( + descriptor: int, + *, + scan: Callable[[int], frozenset[str]] = original_scan, + package_root: Path = root / "netveil", + ) -> frozenset[str]: + nonlocal calls + calls += 1 + observed = scan(descriptor) + if calls == 2: + os.utime(package_root, None) + return observed + + with ( + patch.object( + bootstrap, + "_scan_package_directory", + side_effect=drift, + ), + self.assertRaises(bootstrap._BootstrapFailure), + ): + _verify(root, distribution) + continue + self.assertIsNotNone(locate_override) + with ( + patch.object( + distribution, + "locate_file", + side_effect=locate_override, + ), + self.assertRaises(bootstrap._BootstrapFailure), + ): + _verify(root, distribution) + + def test_loader_preload_spec_and_import_failure_cleanup(self) -> None: + artifact = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=Path("/verified"), + sources=(("netveil", "__init__.py", b"", True),), + ) + with ( + patch.dict(sys.modules, {"netveil": ModuleType("netveil")}), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._load_verified_cli(artifact) + + finder = bootstrap._VerifiedSourceFinder(artifact) + with ( + patch.object(importlib.util, "spec_from_loader", return_value=None), + self.assertRaises(ModuleNotFoundError), + ): + finder.find_spec("netveil", None) + + broken = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=SOURCE_ROOT / "netveil", + sources=( + ( + "netveil", + "__init__.py", + ( + b"import sys, types\n" + b"sys.modules['netveil.partial'] = " + b"types.ModuleType('netveil.partial')\n" + b"raise RuntimeError('broken')\n" + ), + True, + ), + ), + ) + with _isolated_netveil_modules(): + before = list(sys.meta_path) + with self.assertRaises(RuntimeError): + bootstrap._load_verified_cli(broken) + self.assertEqual(sys.meta_path, before) + self.assertNotIn("netveil", sys.modules) + self.assertNotIn("netveil.partial", sys.modules) + + removes_finder = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=SOURCE_ROOT / "netveil", + sources=( + ( + "netveil", + "__init__.py", + b"import sys\nsys.meta_path.pop(0)\nraise RuntimeError('broken')\n", + True, + ), + ), + ) + with _isolated_netveil_modules(): + before = list(sys.meta_path) + with self.assertRaises(RuntimeError): + bootstrap._load_verified_cli(removes_finder) + self.assertEqual(sys.meta_path, before) + + def test_text_writer_oserror_and_successful_main_result(self) -> None: + stream = io.StringIO() + with patch.object(stream, "write", side_effect=OSError("PRIVATE")): + self.assertFalse(bootstrap._write_text(stream, "x")) + + artifact = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=Path("/verified"), + sources=(), + ) + cli = ModuleType("netveil.cli") + + def run( + argv: object, + *, + verified_distribution_version: str, + ) -> int: + self.assertEqual(argv, ["--version"]) + self.assertEqual(verified_distribution_version, "0.3.0") + return 23 + + cli._main_verified = run # type: ignore[attr-defined] + with ( + patch.object( + bootstrap, + "_verify_installed_artifact", + return_value=artifact, + ), + patch.object(bootstrap, "_load_verified_cli", return_value=cli), + ): + self.assertEqual(bootstrap.main(["--version"]), 23) + + def test_bootstrap_main_redacts_failures_and_validates_result_type(self) -> None: + stderr = io.StringIO() + cases = ( + ( + bootstrap._BootstrapFailure("artifact_unverified", 10), + 10, + "artifact_unverified", + ), + (KeyboardInterrupt(), 70, "interrupted"), + (SystemExit(99), 70, "internal_error"), + (RuntimeError("PRIVATE"), 70, "internal_error"), + ) + for failure, expected_exit, expected_code in cases: + with ( + self.subTest(expected_code=expected_code), + patch.object( + bootstrap, + "_verify_installed_artifact", + side_effect=failure, + ), + patch.object(sys, "stderr", stderr), + ): + self.assertEqual(bootstrap.main([]), expected_exit) + self.assertEqual( + stderr.getvalue(), + f"netveil-audit: {expected_code}\n", + ) + stderr.seek(0) + stderr.truncate() + + artifact = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=Path("/verified"), + sources=(), + ) + cli = ModuleType("netveil.cli") + cli._main_verified = ( # type: ignore[attr-defined] + lambda *args, **kwargs: "not-an-int" + ) + with ( + patch.object( + bootstrap, + "_verify_installed_artifact", + return_value=artifact, + ), + patch.object(bootstrap, "_load_verified_cli", return_value=cli), + patch.object(sys, "stderr", stderr), + ): + self.assertEqual(bootstrap.main([]), bootstrap._INTERNAL_FAILURE_EXIT) + self.assertEqual(stderr.getvalue(), "netveil-audit: internal_error\n") + + def test_short_stderr_write_is_bounded(self) -> None: + stderr = io.StringIO() + with ( + patch.object(stderr, "write", return_value=0), + patch.object(sys, "stderr", stderr), + ): + self.assertEqual( + bootstrap._emit_failure("safe", 10), + bootstrap._OUTPUT_FAILURE_EXIT, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..022a639 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,696 @@ +from __future__ import annotations + +import hashlib +import io +import json +import os +import socket +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import TextIO, cast +from unittest.mock import patch + +from netveil import ( + MIN_PSEUDONYMIZATION_KEY_BYTES, + build_privacy_receipt, + cli, +) +from netveil.cli import CliExitCode + +_DEMO_KEY = bytes(range(MIN_PSEUDONYMIZATION_KEY_BYTES)) +_DEMO_CORPUS = ( + b"# IETF documentation ranges only\n" + b"192.0.2.10:443\n" + b"192.0.2.10:443\n" + b"[2001:db8::10]:8443\n" +) + + +class _CapturedStdout: + def __init__(self) -> None: + self.buffer = io.BytesIO() + self.text = io.StringIO() + + def write(self, value: str) -> int: + return self.text.write(value) + + def flush(self) -> None: + self.text.flush() + self.buffer.flush() + + +def _write(path: Path, payload: bytes, mode: int = 0o600) -> None: + path.write_bytes(payload) + path.chmod(mode) + + +class CliWorkflowTests(unittest.TestCase): + def _run_main(self, arguments: list[str] | None) -> tuple[int, bytes, str, str]: + stdout = _CapturedStdout() + stderr = io.StringIO() + with ( + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified( + arguments, + verified_distribution_version=cli._DISTRIBUTION_VERSION, + ) + return ( + result, + stdout.buffer.getvalue(), + stdout.text.getvalue(), + stderr.getvalue(), + ) + + def test_receipt_command_emits_exact_public_bytes_without_network(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "private-corpus.txt" + key_path = root / "private-key.bin" + _write(corpus_path, _DEMO_CORPUS) + _write(key_path, _DEMO_KEY) + + with ( + patch.object( + socket, + "socket", + side_effect=AssertionError("network"), + ), + patch.object( + socket, + "getaddrinfo", + side_effect=AssertionError("resolution"), + ), + patch.object( + subprocess, + "Popen", + side_effect=AssertionError("process"), + ), + patch.object( + subprocess, + "run", + side_effect=AssertionError("process"), + ), + ): + result, binary_output, text_output, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(key_path), + ] + ) + + expected = build_privacy_receipt( + _DEMO_CORPUS, + pseudonymization_key=_DEMO_KEY, + ).canonical_json_bytes() + self.assertEqual(result, CliExitCode.SUCCESS) + self.assertEqual(binary_output, expected + b"\n") + self.assertEqual(text_output, "") + self.assertEqual(error_output, "") + document = json.loads(binary_output) + self.assertEqual(document["schema"], "netveil.aggregate-receipt.v1") + self.assertNotIn("192.0.2.10", binary_output.decode()) + self.assertNotIn(_DEMO_KEY.hex(), binary_output.decode()) + self.assertNotIn( + hashlib.sha256(_DEMO_CORPUS).hexdigest(), binary_output.decode() + ) + + def test_version_is_bound_to_verified_distribution(self) -> None: + result, binary_output, text_output, error_output = self._run_main(["--version"]) + self.assertEqual(result, CliExitCode.SUCCESS) + self.assertEqual(binary_output, b"") + self.assertEqual(text_output, "netveil-audit 0.3.0\n") + self.assertEqual(error_output, "") + with patch.object(sys, "argv", ["netveil-audit", "--version"]): + result, binary_output, text_output, error_output = self._run_main(None) + self.assertEqual(result, CliExitCode.SUCCESS) + self.assertEqual(binary_output, b"") + self.assertEqual(text_output, "netveil-audit 0.3.0\n") + self.assertEqual(error_output, "") + + def test_version_cannot_bypass_or_mix_with_a_workflow(self) -> None: + marker = "PRIVATE-VERSION-MIX-MARKER" + for arguments in ( + ["--version", "--version"], + ["--version", f"--unknown={marker}"], + [ + "--version", + "receipt", + f"{marker}-corpus", + "--key-file", + f"{marker}-key", + ], + ): + with self.subTest(arguments=arguments): + result, binary, text, error = self._run_main(arguments) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(binary, b"") + self.assertEqual(text, "") + self.assertEqual(error, "netveil-audit: usage_error\n") + self.assertNotIn(marker, error) + + def test_key_option_must_appear_exactly_once(self) -> None: + marker = "PRIVATE-REPEATED-KEY-MARKER" + for arguments in ( + [ + "receipt", + f"{marker}-corpus", + "--key-file", + f"{marker}-first", + "--key-file", + f"{marker}-second", + ], + [ + "receipt", + f"{marker}-corpus", + f"--key-file={marker}-first", + "--key-file", + f"{marker}-second", + ], + ["receipt", f"{marker}-corpus", "--key-file="], + ): + with self.subTest(arguments=arguments): + result, binary, text, error = self._run_main(arguments) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(binary, b"") + self.assertEqual(text, "") + self.assertEqual(error, "netveil-audit: usage_error\n") + self.assertNotIn(marker, error) + + def test_long_options_cannot_be_abbreviated(self) -> None: + marker = "PRIVATE-ABBREVIATION-MARKER" + for arguments in ( + ["--ver"], + ["receipt", f"{marker}-corpus", "--key-f", f"{marker}-key"], + ): + with self.subTest(arguments=arguments): + result, binary, text, error = self._run_main(arguments) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(binary, b"") + self.assertEqual(text, "") + self.assertEqual(error, "netveil-audit: usage_error\n") + self.assertNotIn(marker, error) + + def test_invalid_corpus_is_rejected_before_key_access_without_echo(self) -> None: + marker = "PRIVATE-ENDPOINT-MARKER" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / f"{marker}-corpus.txt" + missing_key = root / f"{marker}-missing-key.bin" + _write(corpus_path, f"{marker}:443\n".encode()) + + result, binary_output, text_output, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(missing_key), + ] + ) + + self.assertEqual(result, CliExitCode.CORPUS_REJECTED) + self.assertEqual(binary_output, b"") + self.assertEqual(text_output, "") + self.assertEqual( + error_output, + "netveil-audit: corpus_rejected:invalid_address:line=1\n", + ) + self.assertNotIn(marker, error_output) + + def test_empty_corpus_error_has_no_synthetic_line_number(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "empty.txt" + _write(corpus_path, b"") + + result, _, _, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(root / "unused-key"), + ] + ) + + self.assertEqual(result, CliExitCode.CORPUS_REJECTED) + self.assertEqual( + error_output, + "netveil-audit: corpus_rejected:empty_corpus\n", + ) + + def test_key_access_and_permission_failures_are_stable_and_redacted(self) -> None: + marker = "PRIVATE-KEY-PATH" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "corpus.txt" + _write(corpus_path, _DEMO_CORPUS) + + missing_key = root / f"{marker}-missing" + result, _, _, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(missing_key), + ] + ) + self.assertEqual(result, CliExitCode.KEY_UNAVAILABLE) + self.assertEqual(error_output, "netveil-audit: key_unavailable\n") + self.assertNotIn(marker, error_output) + + insecure_key = root / f"{marker}-insecure" + _write(insecure_key, _DEMO_KEY, 0o644) + result, _, _, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(insecure_key), + ] + ) + self.assertEqual(result, CliExitCode.KEY_REJECTED) + self.assertEqual(error_output, "netveil-audit: key_rejected\n") + self.assertNotIn(marker, error_output) + + def test_corpus_cannot_be_reused_or_copied_as_its_own_key(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "corpus.txt" + copied_key_path = root / "copied-key.bin" + _write(corpus_path, _DEMO_CORPUS) + _write(copied_key_path, _DEMO_CORPUS) + + for key_path in (corpus_path, copied_key_path): + with self.subTest(key_path=key_path.name): + result, binary, text, error = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(key_path), + ] + ) + self.assertEqual(result, CliExitCode.KEY_REJECTED) + self.assertEqual(binary, b"") + self.assertEqual(text, "") + self.assertEqual(error, "netveil-audit: key_rejected\n") + + def test_usage_failure_never_echoes_unrecognized_argument(self) -> None: + marker = "PRIVATE-COMMAND-LINE-MARKER" + result, binary_output, text_output, error_output = self._run_main( + [f"--unknown={marker}"] + ) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(binary_output, b"") + self.assertEqual(text_output, "") + self.assertEqual(error_output, "netveil-audit: usage_error\n") + self.assertNotIn(marker, error_output) + + def test_missing_command_is_a_stable_usage_failure(self) -> None: + result, _, _, error_output = self._run_main([]) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(error_output, "netveil-audit: usage_error\n") + + def test_help_returns_success_without_leaving_main(self) -> None: + result, binary_output, text_output, error_output = self._run_main(["--help"]) + self.assertEqual(result, CliExitCode.SUCCESS) + self.assertEqual(binary_output, b"") + self.assertIn("usage: netveil-audit", text_output) + self.assertEqual(error_output, "") + + def test_artifact_failure_happens_before_argument_or_path_handling(self) -> None: + marker = "PRIVATE-PATH-MARKER" + stdout = _CapturedStdout() + stderr = io.StringIO() + with ( + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified([f"--unknown={marker}"]) + self.assertEqual(result, CliExitCode.ARTIFACT_UNVERIFIED) + self.assertEqual(stderr.getvalue(), "netveil-audit: artifact_unverified\n") + self.assertNotIn(marker, stderr.getvalue()) + + def test_unexpected_failure_and_interrupt_are_redacted(self) -> None: + for failure, expected in ( + (RuntimeError("PRIVATE-INTERNAL-MARKER"), "internal_error"), + (KeyboardInterrupt(), "interrupted"), + ): + with self.subTest(expected=expected): + stdout = _CapturedStdout() + stderr = io.StringIO() + with ( + patch.object( + cli, + "_parser", + side_effect=failure, + ), + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified( + [], + verified_distribution_version=cli._DISTRIBUTION_VERSION, + ) + self.assertEqual(result, CliExitCode.INTERNAL_ERROR) + self.assertEqual(stderr.getvalue(), f"netveil-audit: {expected}\n") + self.assertNotIn("PRIVATE-INTERNAL-MARKER", stderr.getvalue()) + + def test_output_failure_is_redacted(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "corpus.txt" + key_path = root / "key.bin" + _write(corpus_path, _DEMO_CORPUS) + _write(key_path, _DEMO_KEY) + stderr = io.StringIO() + stdout = _CapturedStdout() + with ( + patch.object( + stdout.buffer, + "write", + side_effect=BrokenPipeError("PRIVATE-OUTPUT-MARKER"), + ), + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified( + [ + "receipt", + str(corpus_path), + "--key-file", + str(key_path), + ], + verified_distribution_version=cli._DISTRIBUTION_VERSION, + ) + self.assertEqual(result, CliExitCode.OUTPUT_FAILED) + self.assertEqual(stderr.getvalue(), "netveil-audit: output_failed\n") + self.assertNotIn("PRIVATE-OUTPUT-MARKER", stderr.getvalue()) + + def test_short_version_write_is_an_output_failure(self) -> None: + stdout = _CapturedStdout() + stderr = io.StringIO() + with ( + patch.object(stdout, "write", return_value=0), + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified( + ["--version"], + verified_distribution_version=cli._DISTRIBUTION_VERSION, + ) + self.assertEqual(result, CliExitCode.OUTPUT_FAILED) + self.assertEqual(stderr.getvalue(), "netveil-audit: output_failed\n") + + +@unittest.skipUnless(os.name == "posix", "safe file policy requires POSIX") +class CliFileBoundaryTests(unittest.TestCase): + def test_accepts_exact_regular_owner_only_key(self) -> None: + with tempfile.TemporaryDirectory() as directory: + key_path = Path(directory) / "key.bin" + _write(key_path, _DEMO_KEY, 0o600) + observed = cli._read_bounded_file( + key_path, + maximum_bytes=cli._MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + self.assertEqual(observed.payload, _DEMO_KEY) + + def test_rejects_insecure_short_linked_and_wrong_owner_keys(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cases: list[tuple[str, bytes, int]] = [ + ("short", b"x" * 31, 0o600), + ("group-readable", _DEMO_KEY, 0o640), + ("owner-executable", _DEMO_KEY, 0o700), + ] + for name, payload, mode in cases: + with self.subTest(name=name): + key_path = root / name + _write(key_path, payload, mode) + with self.assertRaises(cli._CliFailure) as raised: + cli._read_bounded_file( + key_path, + maximum_bytes=cli._MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + self.assertEqual(raised.exception.code, "key_rejected") + key_path.unlink() + + linked = root / "linked" + alias = root / "alias" + _write(linked, _DEMO_KEY) + os.link(linked, alias) + with self.assertRaises(cli._CliFailure) as linked_failure: + cli._read_bounded_file( + linked, + maximum_bytes=cli._MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + self.assertEqual(linked_failure.exception.code, "key_rejected") + + linked.unlink() + alias.unlink() + wrong_owner = root / "wrong-owner" + _write(wrong_owner, _DEMO_KEY) + current_uid = wrong_owner.stat().st_uid + with ( + patch.object(os, "geteuid", return_value=current_uid + 1), + self.assertRaises(cli._CliFailure) as owner_failure, + ): + cli._read_bounded_file( + wrong_owner, + maximum_bytes=cli._MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + self.assertEqual(owner_failure.exception.code, "key_rejected") + + def test_rejects_symlink_directory_fifo_missing_and_oversized_files(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / "target" + _write(target, _DEMO_KEY) + symlink = root / "symlink" + symlink.symlink_to(target) + fifo = root / "fifo" + os.mkfifo(fifo) + oversized = root / "oversized" + _write(oversized, b"x" * 33) + + for name, path, maximum in ( + ("symlink", symlink, cli._MAX_KEY_BYTES), + ("directory", root, cli._MAX_KEY_BYTES), + ("fifo", fifo, cli._MAX_KEY_BYTES), + ("missing", root / "missing", cli._MAX_KEY_BYTES), + ("oversized", oversized, 32), + ): + with ( + self.subTest(name=name), + self.assertRaises(cli._CliFailure) as raised, + ): + cli._read_bounded_file( + path, + maximum_bytes=maximum, + failure_code="file_unavailable", + failure_exit=CliExitCode.CORPUS_UNAVAILABLE, + key_policy=False, + ) + self.assertEqual(raised.exception.code, "file_unavailable") + self.assertIsNone(raised.exception.__context__) + + def test_detects_file_identity_change_and_close_failure(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input" + _write(path, b"abc") + status = path.stat() + changed = os.stat_result( + ( + status.st_mode, + status.st_ino, + status.st_dev, + status.st_nlink, + status.st_uid, + status.st_gid, + status.st_size + 1, + status.st_atime, + status.st_mtime, + status.st_ctime, + ) + ) + with ( + patch.object( + cli, + "_read_open_file", + return_value=(b"abc", status, changed), + ), + self.assertRaises(cli._CliFailure), + ): + cli._read_bounded_file( + path, + maximum_bytes=10, + failure_code="changed", + failure_exit=CliExitCode.CORPUS_UNAVAILABLE, + key_policy=False, + ) + + real_close = os.close + + def close_then_fail(descriptor: int) -> None: + real_close(descriptor) + raise OSError("PRIVATE-CLOSE-MARKER") + + with ( + patch.object(os, "close", side_effect=close_then_fail), + self.assertRaises(cli._CliFailure) as close_failure, + ): + cli._read_bounded_file( + path, + maximum_bytes=10, + failure_code="close_failed", + failure_exit=CliExitCode.CORPUS_UNAVAILABLE, + key_policy=False, + ) + self.assertEqual(close_failure.exception.code, "close_failed") + self.assertIsNone(close_failure.exception.__context__) + + def test_low_level_read_failure_returns_only_empty_sentinels(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input" + _write(path, b"abc") + descriptor = os.open(path, os.O_RDONLY) + try: + with patch.object(os, "fstat", side_effect=OSError("PRIVATE")): + result = cli._read_open_file(descriptor, maximum_bytes=10) + finally: + os.close(descriptor) + self.assertEqual(result, (None, None, None)) + + def test_low_level_read_stops_at_bound_without_waiting_for_eof(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input" + _write(path, b"abc") + descriptor = os.open(path, os.O_RDONLY) + try: + current = os.fstat(descriptor) + before_growth = os.stat_result( + ( + current.st_mode, + current.st_ino, + current.st_dev, + current.st_nlink, + current.st_uid, + current.st_gid, + 2, + current.st_atime, + current.st_mtime, + current.st_ctime, + ) + ) + with patch.object( + os, + "fstat", + side_effect=(before_growth, current), + ): + payload, before, after = cli._read_open_file( + descriptor, + maximum_bytes=2, + ) + finally: + os.close(descriptor) + self.assertEqual(payload, b"abc") + self.assertIsNotNone(before) + self.assertIsNotNone(after) + + def test_unsupported_platform_fails_before_open(self) -> None: + with ( + patch.object(os, "name", "unsupported"), + self.assertRaises(cli._CliFailure) as raised, + ): + cli._open_flags() + self.assertEqual(raised.exception.code, "platform_unsupported") + + +class CliOutputBoundaryTests(unittest.TestCase): + def test_emit_failure_survives_broken_stderr(self) -> None: + stderr = io.StringIO() + with ( + patch.object( + stderr, + "write", + side_effect=OSError("PRIVATE-STDERR-MARKER"), + ), + patch.object(sys, "stderr", stderr), + ): + result = cli._emit_failure( + cli._CliFailure("safe", CliExitCode.CORPUS_REJECTED) + ) + self.assertEqual(result, CliExitCode.OUTPUT_FAILED) + + def test_parser_exit_messages_use_bounded_writes(self) -> None: + parser = cli._SafeArgumentParser(prog="netveil-audit") + stderr = io.StringIO() + with ( + patch.object(stderr, "write", return_value=0), + self.assertRaises(cli._CliFailure) as output_failure, + ): + parser._print_message("help", stderr) + self.assertEqual(output_failure.exception.code, "output_failed") + + with ( + patch.object(sys, "stderr", stderr), + self.assertRaises(cli._CliFailure) as usage_failure, + ): + parser.exit(2, "usage failed\n") + self.assertEqual(usage_failure.exception.code, "usage_error") + + stdout = io.StringIO() + with ( + patch.object(sys, "stdout", stdout), + self.assertRaises(cli._CliCompletion) as completion, + ): + parser.exit(0, "done\n") + self.assertEqual(completion.exception.exit_code, CliExitCode.SUCCESS) + self.assertEqual(stdout.getvalue(), "done\n") + + def test_exact_writers_reject_zero_and_accept_partial_progress(self) -> None: + text = io.StringIO() + binary = io.BytesIO() + with patch.object( + text, + "write", + side_effect=lambda value: min(2, len(value)), + ) as text_write: + self.assertTrue(cli._write_text(text, "abcdef")) + with patch.object( + binary, + "write", + side_effect=lambda value: min(2, len(value)), + ) as binary_write: + self.assertTrue(cli._write_binary(binary, b"abcdef")) + self.assertEqual(text_write.call_count, 3) + self.assertEqual(binary_write.call_count, 3) + + with ( + patch.object(text, "write", return_value=0), + patch.object(binary, "write", return_value=0), + ): + self.assertFalse(cli._write_text(text, "x")) + self.assertFalse(cli._write_binary(binary, b"x")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_launcher.py b/tests/test_launcher.py new file mode 100644 index 0000000..eecdb28 --- /dev/null +++ b/tests/test_launcher.py @@ -0,0 +1,568 @@ +from __future__ import annotations + +import base64 +import contextlib +import hashlib +import io +import os +import stat +import sys +import tempfile +import unittest +from collections.abc import Iterator +from importlib import metadata +from pathlib import Path +from types import ModuleType +from typing import cast +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +LAUNCHER_PATH = ROOT / "scripts" / "netveil-audit" +BOOTSTRAP_FIXTURE = ROOT / "tests" / "fixtures" / "launcher_bootstrap_fixture.py" + + +class _FakeHash: + def __init__(self, value: str, mode: str = "sha256") -> None: + self.mode = mode + self.value = value + + +class _FakeRecord: + def __init__( + self, + payload: bytes, + *, + path: str = "artifact", + mode: str = "sha256", + size: int | None = None, + ) -> None: + self.path = path + digest = hashlib.sha256(payload).digest() + value = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + self.hash: _FakeHash | None = _FakeHash(value, mode) + self.size = len(payload) if size is None else size + + def __str__(self) -> str: + return self.path + + +class _FakeDistribution: + def __init__( + self, + root: Path, + files: list[_FakeRecord] | None, + ) -> None: + self.root = root + self.files = files + self.metadata = { + "Name": "netveil-audit", + "Version": "0.3.0", + } + self.locations: dict[str, Path] = {} + + def locate_file(self, record: object) -> Path: + raw_path = str(record) + return self.locations.get(raw_path, self.root / raw_path) + + +def _load_launcher() -> ModuleType: + module = ModuleType("netveil_launcher_test") + module.__file__ = str(LAUNCHER_PATH) + source = LAUNCHER_PATH.read_bytes() + exec( # noqa: S102 - test loads the repository-owned launcher source. + compile(source, str(LAUNCHER_PATH), "exec", dont_inherit=True), + module.__dict__, + ) + return module + + +@contextlib.contextmanager +def _without_netveil_modules() -> Iterator[None]: + saved = { + name: module + for name, module in sys.modules.items() + if name == "netveil_bootstrap" + or name == "netveil" + or name.startswith("netveil.") + } + for name in saved: + sys.modules.pop(name, None) + try: + yield + finally: + for name in tuple(sys.modules): + if ( + name == "netveil_bootstrap" + or name == "netveil" + or name.startswith("netveil.") + ): + sys.modules.pop(name, None) + sys.modules.update(saved) + + +class LauncherBoundaryTests(unittest.TestCase): + def setUp(self) -> None: + self.launcher = _load_launcher() + self.bootstrap_payload = BOOTSTRAP_FIXTURE.read_bytes() + + def test_polyglot_header_requests_exact_isolation_profile(self) -> None: + lines = LAUNCHER_PATH.read_text().splitlines() + self.assertEqual(lines[0], "#!/bin/sh") + self.assertEqual( + lines[6], + 'exec "$netveil_script_directory/python" -IESB "$0" "$@"', + ) + self.assertFalse(self.launcher._startup_is_isolated()) + + def test_python_main_guard_fails_closed_without_isolation(self) -> None: + namespace = { + "__file__": str(LAUNCHER_PATH), + "__name__": "__main__", + } + stderr = io.StringIO() + with ( + patch.object(sys, "stderr", stderr), + self.assertRaises(SystemExit) as raised, + ): + exec( # noqa: S102 - test executes repository-owned launcher source. + compile( + LAUNCHER_PATH.read_bytes(), + str(LAUNCHER_PATH), + "exec", + dont_inherit=True, + ), + namespace, + ) + self.assertEqual(raised.exception.code, 10) + self.assertEqual(stderr.getvalue(), "netveil-audit: artifact_unverified\n") + + def test_site_root_is_bound_to_installed_prefix_layout(self) -> None: + with tempfile.TemporaryDirectory() as directory: + prefix = Path(directory) + launcher = prefix / "bin" / "netveil-audit" + launcher.parent.mkdir() + launcher.write_bytes(b"launcher") + site_root = ( + prefix + / "lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "site-packages" + ) + (site_root / "netveil_audit-0.3.0.dist-info").mkdir(parents=True) + self.assertEqual(self.launcher._site_root(launcher), site_root) + + duplicate = ( + prefix + / "lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "dist-packages" + / "netveil_audit-0.3.0.dist-info" + ) + duplicate.mkdir(parents=True) + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._site_root(launcher) + + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._site_root(Path("/tmp/netveil-audit")) + + def test_record_reader_pins_exact_regular_bytes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "artifact" + payload = b"record-bound" + path.write_bytes(payload) + record = _FakeRecord(payload) + observed, identity = self.launcher._read_record_bound(path, record) + self.assertEqual(observed, payload) + self.assertEqual(identity, self.launcher._identity(path.stat())) + + path.write_bytes(b"record-b0und") + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_record_bound(path, record) + + def test_record_reader_rejects_contract_and_file_shape_failures(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "artifact" + path.write_bytes(b"x") + for record in ( + _FakeRecord(b"x", mode="sha512"), + _FakeRecord(b"x", size=True), + _FakeRecord(b"x", size=-1), + _FakeRecord( + b"x", + size=self.launcher._MAX_ARTIFACT_FILE_BYTES + 1, + ), + ): + with ( + self.subTest(record=record), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + + no_hash = _FakeRecord(b"x") + no_hash.hash = None + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_record_bound(path, no_hash) + + directory_path = root / "directory" + directory_path.mkdir() + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_record_bound( + directory_path, + _FakeRecord(b"", size=0), + ) + + symlink = root / "link" + symlink.symlink_to(path) + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_record_bound(symlink, _FakeRecord(b"x")) + + oversized = root / "oversized" + oversized.write_bytes(b"x" * (self.launcher._MAX_ARTIFACT_FILE_BYTES + 1)) + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_bounded_regular(oversized) + + def test_record_reader_handles_io_and_close_failures(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "artifact" + path.write_bytes(b"x") + record = _FakeRecord(b"x") + with ( + patch.object(self.launcher.os, "open", side_effect=OSError), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + with ( + patch.object(self.launcher.os, "read", side_effect=OSError), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + with ( + patch.object(self.launcher.os, "close", side_effect=OSError), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + with ( + patch.object(self.launcher.os, "read", return_value=b"xx"), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + + def _installed_artifact( + self, + root: Path, + ) -> tuple[Path, Path, _FakeDistribution, bytes]: + prefix = root / "prefix" + launcher_path = prefix / "bin" / "netveil-audit" + launcher_path.parent.mkdir(parents=True) + launcher_payload = b"trusted launcher" + launcher_path.write_bytes(launcher_payload) + site_root = ( + prefix + / "lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "site-packages" + ) + dist_info = site_root / "netveil_audit-0.3.0.dist-info" + dist_info.mkdir(parents=True) + (dist_info / "METADATA").write_bytes(b"bounded metadata") + (dist_info / "RECORD").write_bytes(b"bounded record") + bootstrap_payload = b"trusted bootstrap" + (site_root / "netveil_bootstrap.py").write_bytes(bootstrap_payload) + launcher_record_path = os.path.relpath(launcher_path, site_root) + records = [ + _FakeRecord(launcher_payload, path=launcher_record_path), + _FakeRecord(bootstrap_payload, path="netveil_bootstrap.py"), + ] + return ( + launcher_path, + site_root, + _FakeDistribution(site_root, records), + bootstrap_payload, + ) + + def test_verified_bootstrap_binds_launcher_and_source_records(self) -> None: + with tempfile.TemporaryDirectory() as directory: + launcher_path, site_root, distribution, expected = self._installed_artifact( + Path(directory) + ) + original_path = list(sys.path) + try: + with patch.object( + self.launcher.importlib.metadata, + "PathDistribution", + return_value=cast(metadata.Distribution, distribution), + ): + payload, bootstrap_path = self.launcher._verified_bootstrap( + launcher_path + ) + finally: + sys.path[:] = original_path + self.assertEqual(payload, expected) + self.assertEqual(bootstrap_path, site_root / "netveil_bootstrap.py") + + def test_verified_bootstrap_rejects_metadata_and_record_drift(self) -> None: + mutations = ( + "missing-metadata", + "wrong-name", + "wrong-version", + "missing-launcher", + "missing-bootstrap", + "entry-points", + ) + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + launcher_path, _, distribution, _ = self._installed_artifact( + Path(directory) + ) + assert distribution.files is not None + if mutation == "missing-metadata": + distribution.metadata.pop("Name") + elif mutation == "wrong-name": + distribution.metadata["Name"] = "other" + elif mutation == "wrong-version": + distribution.metadata["Version"] = "9.9.9" + elif mutation == "missing-launcher": + distribution.files = [ + record + for record in distribution.files + if str(record) == "netveil_bootstrap.py" + ] + elif mutation == "missing-bootstrap": + distribution.files = [ + record + for record in distribution.files + if str(record) != "netveil_bootstrap.py" + ] + else: + site_root = distribution.root + ( + site_root / "netveil_audit-0.3.0.dist-info" / "entry_points.txt" + ).write_bytes(b"[console_scripts]\n") + with ( + patch.object( + self.launcher.importlib.metadata, + "PathDistribution", + return_value=cast(metadata.Distribution, distribution), + ), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._verified_bootstrap(launcher_path) + + def test_record_inventory_and_terminal_identity_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + _, _, distribution, _ = self._installed_artifact(Path(directory)) + distribution.files = None + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._records(cast(metadata.Distribution, distribution)) + + duplicate = _FakeRecord(b"x", path="same") + distribution = _FakeDistribution(Path("/unused"), [duplicate, duplicate]) + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._records(cast(metadata.Distribution, distribution)) + + with tempfile.TemporaryDirectory() as directory: + launcher_path, _, distribution, _ = self._installed_artifact( + Path(directory) + ) + with ( + patch.object( + self.launcher.os, + "lstat", + side_effect=PermissionError, + ), + patch.object( + self.launcher.importlib.metadata, + "PathDistribution", + return_value=cast(metadata.Distribution, distribution), + ), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._verified_bootstrap(launcher_path) + + for mismatch_call in (2, 4): + with ( + self.subTest(mismatch_call=mismatch_call), + tempfile.TemporaryDirectory() as directory, + ): + launcher_path, _, distribution, _ = self._installed_artifact( + Path(directory) + ) + reads = [ + (b"launcher", (1,)), + (b"launcher", (1 if mismatch_call != 2 else 2,)), + (b"bootstrap", (3,)), + (b"bootstrap", (3 if mismatch_call != 4 else 4,)), + ] + with ( + patch.object( + self.launcher.importlib.metadata, + "PathDistribution", + return_value=cast(metadata.Distribution, distribution), + ), + patch.object( + self.launcher, + "_read_record_bound", + side_effect=reads, + ), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._verified_bootstrap(launcher_path) + + def test_verified_bootstrap_bytes_are_executed_in_memory(self) -> None: + with ( + _without_netveil_modules(), + patch.object(sys, "argv", ["netveil-audit", "safe"]), + ): + self.assertEqual( + self.launcher._execute_bootstrap( + self.bootstrap_payload, + BOOTSTRAP_FIXTURE, + ), + 23, + ) + + def test_preload_invalid_result_and_exception_fail_closed(self) -> None: + with ( + patch.dict(sys.modules, {"netveil": ModuleType("netveil")}), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._execute_bootstrap( + self.bootstrap_payload, + BOOTSTRAP_FIXTURE, + ) + + with ( + _without_netveil_modules(), + patch.object( + sys, + "argv", + ["netveil-audit", "invalid"], + ), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._execute_bootstrap(self.bootstrap_payload, BOOTSTRAP_FIXTURE) + + with _without_netveil_modules(): + with ( + patch.object(sys, "argv", ["netveil-audit", "explode"]), + self.assertRaises(RuntimeError), + ): + self.launcher._execute_bootstrap( + self.bootstrap_payload, + BOOTSTRAP_FIXTURE, + ) + self.assertNotIn("netveil_bootstrap", sys.modules) + + marker = "PRIVATE-LAUNCHER-MARKER" + stderr = io.StringIO() + with ( + patch.object( + self.launcher, + "_startup_is_isolated", + side_effect=RuntimeError(marker), + ), + patch.object(sys, "stderr", stderr), + ): + self.assertEqual( + self.launcher.main(), + self.launcher._INTERNAL_FAILURE_EXIT, + ) + self.assertEqual(stderr.getvalue(), "netveil-audit: internal_error\n") + self.assertNotIn(marker, stderr.getvalue()) + + def test_exact_error_writer_handles_partial_and_broken_output(self) -> None: + class PartialWriter(io.StringIO): + def write(self, value: str) -> int: + return super().write(value[:1]) + + writer = PartialWriter() + with patch.object(sys, "stderr", writer): + self.assertEqual( + self.launcher._write_failure("safe", 23), + 23, + ) + self.assertEqual(writer.getvalue(), "netveil-audit: safe\n") + + for written in (0, None, 100): + broken = io.StringIO() + with ( + patch.object(broken, "write", return_value=written), + patch.object(sys, "stderr", broken), + ): + self.assertEqual( + self.launcher._write_failure("safe", 23), + self.launcher._OUTPUT_FAILURE_EXIT, + ) + with patch.object(sys.stderr, "write", side_effect=OSError): + self.assertEqual( + self.launcher._write_failure("safe", 23), + self.launcher._OUTPUT_FAILURE_EXIT, + ) + + def test_file_flags_and_identity_are_fail_closed(self) -> None: + required = os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + self.assertEqual(self.launcher._file_flags() & required, required) + status = LAUNCHER_PATH.stat() + self.assertTrue(stat.S_ISREG(status.st_mode)) + self.assertEqual( + self.launcher._identity(status), + ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_uid, + status.st_gid, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ), + ) + with ( + patch("builtins.hasattr", return_value=False), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._file_flags() + + def test_main_maps_success_and_failures_to_stable_codes(self) -> None: + stderr = io.StringIO() + with ( + patch.object(sys, "stderr", stderr), + patch.object(self.launcher, "_startup_is_isolated", return_value=False), + ): + self.assertEqual( + self.launcher.main(), + self.launcher._ARTIFACT_FAILURE_EXIT, + ) + self.assertEqual(stderr.getvalue(), "netveil-audit: artifact_unverified\n") + + with ( + patch.object(self.launcher, "_startup_is_isolated", return_value=True), + patch.object( + self.launcher, + "_verified_bootstrap", + return_value=(b"source", Path("/bootstrap")), + ), + patch.object(self.launcher, "_execute_bootstrap", return_value=23), + ): + self.assertEqual(self.launcher.main(), 23) + + stderr = io.StringIO() + with ( + patch.object(sys, "stderr", stderr), + patch.object(self.launcher, "_startup_is_isolated", return_value=True), + patch.object( + self.launcher, + "_verified_bootstrap", + side_effect=KeyboardInterrupt, + ), + ): + self.assertEqual( + self.launcher.main(), + self.launcher._INTERNAL_FAILURE_EXIT, + ) + self.assertEqual(stderr.getvalue(), "netveil-audit: interrupted\n") diff --git a/tests/test_packaging.py b/tests/test_packaging.py index cbe0b83..1001941 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -1,7 +1,12 @@ from __future__ import annotations +import stat +import tomllib import unittest from importlib import resources +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] class PackagingContractTests(unittest.TestCase): @@ -10,6 +15,39 @@ def test_pep561_marker_is_packaged(self) -> None: self.assertTrue(marker.is_file()) self.assertIn(marker.read_bytes(), (b"", b"\n")) + def test_static_launcher_replaces_generated_entry_point(self) -> None: + document = tomllib.loads((ROOT / "pyproject.toml").read_text()) + project = document["project"] + setuptools = document["tool"]["setuptools"] + self.assertNotIn("scripts", project) + self.assertEqual(project["dependencies"], []) + self.assertEqual(setuptools["script-files"], ["scripts/netveil-audit"]) + self.assertEqual(setuptools["py-modules"], ["netveil_bootstrap"]) + + launcher = ROOT / "scripts" / "netveil-audit" + self.assertEqual(launcher.read_bytes().splitlines()[0], b"#!/bin/sh") + self.assertTrue(launcher.stat().st_mode & stat.S_IXUSR) + + def test_source_manifest_keeps_release_evidence_reproducible(self) -> None: + manifest = (ROOT / "MANIFEST.in").read_text().splitlines() + required = { + "include README.md", + "include SECURITY.md", + "include requirements-dev.txt", + "include scripts/netveil-audit", + "recursive-include docs *.md", + "recursive-include docs/assets *.svg *.json *.cast *.gif", + "recursive-include tests *.py", + "recursive-include tools *.py", + } + self.assertTrue(required.issubset(manifest)) + + def test_developer_tool_versions_are_exactly_pinned(self) -> None: + requirements = (ROOT / "requirements-dev.txt").read_text().splitlines() + pins = [line for line in requirements if line and not line.startswith("#")] + self.assertGreaterEqual(len(pins), 4) + self.assertTrue(all(line.count("==") == 1 for line in pins)) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_visuals.py b/tests/test_visuals.py new file mode 100644 index 0000000..87486f1 --- /dev/null +++ b/tests/test_visuals.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + +from tools import render_architecture + +ROOT = Path(__file__).resolve().parents[1] + + +class VisualEvidenceTests(unittest.TestCase): + def test_architecture_svg_is_current_and_code_derived(self) -> None: + expected = render_architecture.render() + observed = (ROOT / "docs" / "assets" / "architecture.svg").read_bytes() + self.assertEqual(observed, expected) + text = observed.decode("utf-8") + self.assertIn("Netveil installed execution boundary", text) + self.assertIn("-I -E -S -B", text) + self.assertIn("installed RECORD is not a signature", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..dd9ff35 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ +"""Repository-local verification tools for Netveil.""" diff --git a/tools/render_architecture.py b/tools/render_architecture.py new file mode 100755 index 0000000..0ac6509 --- /dev/null +++ b/tools/render_architecture.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Render the code-derived Netveil execution-boundary diagram.""" + +from __future__ import annotations + +import argparse +import ast +import html +from pathlib import Path +from typing import Final + +ROOT: Final = Path(__file__).resolve().parents[1] +OUTPUT: Final = ROOT / "docs" / "assets" / "architecture.svg" + + +def _integer(expression: ast.expr) -> int: + if isinstance(expression, ast.Constant) and type(expression.value) is int: + return expression.value + if isinstance(expression, ast.BinOp) and isinstance(expression.op, ast.Mult): + return _integer(expression.left) * _integer(expression.right) + raise ValueError("unsupported integer constant") + + +def _assignments(path: Path) -> dict[str, ast.expr]: + tree = ast.parse(path.read_bytes(), filename=str(path)) + values: dict[str, ast.expr] = {} + for node in tree.body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + values[target.id] = node.value + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if node.value is not None: + values[node.target.id] = node.value + return values + + +def _source_contract() -> tuple[str, int, int, int, tuple[str, ...]]: + launcher = (ROOT / "scripts" / "netveil-audit").read_text() + handoff = 'exec "$netveil_script_directory/python" -IESB "$0" "$@"' + if handoff not in launcher: + raise ValueError("launcher isolation handoff changed") + + parser = _assignments(ROOT / "src" / "netveil" / "parser.py") + cli = _assignments(ROOT / "src" / "netveil" / "cli.py") + bootstrap = _assignments(ROOT / "src" / "netveil_bootstrap.py") + source_modules = ast.literal_eval(bootstrap["_SOURCE_MODULES"]) + module_names = tuple(module for module, _, _ in source_modules) + return ( + "-I -E -S -B", + _integer(parser["MAX_INPUT_BYTES"]), + _integer(parser["MAX_PHYSICAL_LINES"]), + _integer(cli["_MAX_KEY_BYTES"]), + module_names, + ) + + +def _text( + x: int, + y: int, + value: str, + *, + size: int = 16, + weight: int = 400, + fill: str = "#d8e3f0", + anchor: str = "start", +) -> str: + return ( + f'{html.escape(value)}' + ) + + +def _box( + x: int, + y: int, + width: int, + height: int, + *, + fill: str, + stroke: str, + title: str, + lines: tuple[str, ...], +) -> str: + content = [ + ( + f'' + ), + _text(x + 22, y + 34, title, size=18, weight=700, fill="#f8fbff"), + ] + for index, line in enumerate(lines): + content.append( + _text( + x + 22, + y + 64 + index * 24, + line, + size=14, + fill="#c7d3e0", + ) + ) + return "\n".join(content) + + +def render() -> bytes: + flags, max_bytes, max_lines, max_key, modules = _source_contract() + module_label = ", ".join(modules) + mib = max_bytes // (1024 * 1024) + svg = [ + ( + '' + ), + 'Netveil installed execution and trust boundary', + ( + 'Code-derived architecture from the installed ' + "polyglot launcher through pinned source loading to a pseudonymized " + "receipt." + ), + "", + ( + '' + '' + '' + "" + ), + ( + '' + '' + "" + ), + "", + '', + _text(70, 66, "Netveil installed execution boundary", size=34, weight=750), + _text( + 70, + 99, + "Generated from the 0.3.0 launcher, parser, CLI and bootstrap sources", + size=16, + fill="#8fa7bd", + ), + _box( + 70, + 145, + 390, + 170, + fill="#33251a", + stroke="#f59e0b", + title="1 Trusted startup", + lines=( + "/bin/sh → sibling CPython", + f"combined flags: {flags}", + "no site hooks • env ignored • safe path", + "launcher + installed RECORD are trust roots", + ), + ), + _box( + 525, + 145, + 390, + 170, + fill="#142a32", + stroke="#22d3ee", + title="2 Artifact drift guard", + lines=( + "launcher and bootstrap: installed hash + size", + "bounded O_NOFOLLOW descriptor reads", + "identity checked before and after read", + "installed RECORD is not a signature", + ), + ), + _box( + 980, + 145, + 390, + 170, + fill="#17273a", + stroke="#60a5fa", + title="3 Closed source loader", + lines=( + f"{len(modules)} allowed modules", + "pinned source bytes compiled in memory", + "unknown netveil.* imports rejected", + "package bytecode never executes", + ), + ), + ( + '' + ), + ( + '' + ), + _box( + 70, + 395, + 600, + 205, + fill="#1f2038", + stroke="#a78bfa", + title="4 Sensitive local inputs", + lines=( + f"corpus: regular file • ≤ {mib} MiB • ≤ {max_lines:,} lines", + f"key: owner-only regular file • 32–{max_key:,} exact bytes", + "same inode or byte-identical corpus/key rejected", + "sensitive values omitted from handled output", + "no DNS, socket or child-process workflow", + ), + ), + _box( + 770, + 395, + 600, + 205, + fill="#173126", + stroke="#34d399", + title="5 Public deterministic result", + lines=( + "parse → canonicalize → aggregate", + "domain-separated HMAC-SHA256 identifiers", + "fixed-schema sorted-key canonical JSON", + "receipt digest binds the public report", + "pseudonymized, not anonymous", + ), + ), + ( + '' + ), + ( + '' + ), + ( + '' + ), + _text(94, 704, "Exact source inventory", size=18, weight=700, fill="#f8fbff"), + _text(94, 739, module_label, size=14, fill="#9fb7cc"), + _text(94, 792, "Trust-root legend", size=16, weight=700, fill="#fbbf24"), + _text( + 94, + 823, + "amber = executes before or supplies the mutable consistency boundary", + size=14, + fill="#c7d3e0", + ), + _text(760, 792, "Verified-byte legend", size=16, weight=700, fill="#67e8f9"), + _text( + 760, + 823, + "cyan / blue = checked before compilation by the supported installed path", + size=14, + fill="#c7d3e0", + ), + _text( + 70, + 935, + "Reproduce: python3 tools/render_architecture.py --check", + size=14, + fill="#6f8aa3", + ), + _text( + 1370, + 935, + "No network data or endpoint fixture is used by this diagram", + size=14, + fill="#6f8aa3", + anchor="end", + ), + "", + ] + return ("\n".join(svg) + "\n").encode() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + action="store_true", + help="fail when the committed SVG differs from generated bytes", + ) + arguments = parser.parse_args() + payload = render() + if arguments.check: + if not OUTPUT.is_file() or OUTPUT.read_bytes() != payload: + raise SystemExit("architecture.svg is stale") + return 0 + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + OUTPUT.write_bytes(payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From f3e4db737c4fd87b4b58f842a926ebb0f9889e69 Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:15:49 +0000 Subject: [PATCH 6/7] Bind releases to clean source artifacts --- README.md | 58 + SECURITY.md | 7 + docs/artifact-boundary.md | 20 + pyproject.toml | 15 + tests/test_evidence_renderer.py | 325 ++++ tests/test_fresh_wheel_verifier.py | 924 +++++++++++ tests/test_packaging.py | 11 + tests/test_release_builder.py | 560 +++++++ tools/build_release.py | 821 +++++++++ tools/render_evidence.py | 1220 ++++++++++++++ tools/verify_fresh_wheel.py | 2472 ++++++++++++++++++++++++++++ 11 files changed, 6433 insertions(+) create mode 100644 tests/test_evidence_renderer.py create mode 100644 tests/test_fresh_wheel_verifier.py create mode 100644 tests/test_release_builder.py create mode 100755 tools/build_release.py create mode 100755 tools/render_evidence.py create mode 100755 tools/verify_fresh_wheel.py diff --git a/README.md b/README.md index 11cb918..c8034d4 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,64 @@ environment: it accepts an exact wheel, installs it into a fresh venv with `--no-index --no-deps`, and exercises startup injection, bytecode, tamper, determinism, and redaction cases. +## Reproducible release evidence + +The publication gate requires Git, CPython 3.12, the pinned development +packages above, and Linux `strace`. Run it only from a clean checkout. The +builder enforces a clean Git top-level, exports the exact `HEAD` with +`git archive`, and rejects an output directory inside the source tree. + +```bash +set -eu + +COMMIT=$(git rev-parse --verify 'HEAD^{commit}') +SOURCE_DATE_EPOCH=$(git show -s --format=%ct "$COMMIT") +OUT="$(dirname "$PWD")/netveil-release-${COMMIT}" +test ! -e "$OUT" + +.venv/bin/python tools/build_release.py \ + "$PWD" "$OUT" \ + --source-date-epoch "$SOURCE_DATE_EPOCH" \ + --python "$PWD/.venv/bin/python" + +VERIFY_TMP="$OUT/.fresh-wheel-verification.json.tmp" +.venv/bin/python tools/verify_fresh_wheel.py \ + --source-commit "$COMMIT" \ + --inventory "$OUT/release-inventory.json" \ + --sdist "$OUT/netveil_audit-0.3.0.tar.gz" \ + "$OUT/netveil_audit-0.3.0-py3-none-any.whl" > "$VERIFY_TMP" +mv "$VERIFY_TMP" "$OUT/fresh-wheel-verification.json" +``` + +The verifier reads the actual wheel, sdist, and canonical inventory; installs +the pinned wheel into a fresh no-dependency environment; runs startup, +bytecode, tamper, deterministic-output, redaction, public-demo, and syscall +checks; and emits one path-free canonical JSON line. + +To refresh the committed evidence views after a successful exact-commit run: + +```bash +mkdir -p docs/evidence +cp -- "$OUT/release-inventory.json" \ + docs/evidence/release-inventory.json +cp -- "$OUT/fresh-wheel-verification.json" \ + docs/evidence/fresh-wheel-verification.json + +.venv/bin/python tools/render_evidence.py +.venv/bin/python tools/render_evidence.py --check +``` + +Repeat the build into a second nonexistent sibling directory and compare the +wheel, sdist, and inventory bytes when testing build reproducibility. + +These JSON files are unsigned consistency and execution evidence, not a +signature or remote attestation. Their trusted computing base includes the +checked-out verifier, Git and `git archive`, CPython, `build`, setuptools, +pip/ensurepip, `strace`, the OS, and filesystem. They cannot prove that the +verifier or host was honest. Publish artifact digests through a separately +authenticated channel. The recorded trace hash covers normalized path-free +facts, not the temporary raw `strace` bytes. + ## Deliberate exclusions Netveil is not a scanner, proxy checker, service-discovery client, reachability diff --git a/SECURITY.md b/SECURITY.md index f92f4f1..32dded3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -123,6 +123,13 @@ one exact artifact and runtime. That trace is evidence for the recorded run, not a guarantee about a compromised trusted interpreter, shell, operating system, or preload mechanism. +Release inventory and fresh-wheel verification JSON are unsigned. They bind a +builder-observed clean commit to exact artifact and executed-test facts, but +they do not authenticate the publisher, attest the build host, or prove that +Git, the verifier, build backend, installer, `strace`, OS, and filesystem were +honest. Obtain published artifact digests through a separately authenticated +channel. + Netveil is not a scanner or reachability tester. Historical Git commits contain unverified public endpoint strings removed from the current tree. Their existence does not grant permission to connect to, probe, or test those diff --git a/docs/artifact-boundary.md b/docs/artifact-boundary.md index 3f6bb64..3b18e6e 100644 --- a/docs/artifact-boundary.md +++ b/docs/artifact-boundary.md @@ -153,3 +153,23 @@ wheel installation. The gate must bind: Coordinated code-plus-`RECORD` mutations belong in negative trust-boundary evidence: they must be shown as accepted, not misleadingly presented as attacks the unsigned guard prevents. + +## Release-evidence trust boundary + +The release builder and verifier establish an internally consistent chain: +clean Git `HEAD` to `git archive`, wheel and canonical sdist bytes, fresh +installation, selected installed files, executed checks, and normalized +process/network trace facts. The release inventory also carries the exact +safe sdist member inventory. + +That chain is unsigned. It trusts the checked-out builder, verifier, and +renderer sources; Git and `git archive`; CPython, `build`, and setuptools; +pip and ensurepip; `strace`; the operating system and filesystem; and the +channel used to publish the resulting digests. It does not prove that those +components were unmodified, that the build or CI host was honest, or that the +person publishing the manifest is Omar. Publisher authentication remains an +external release-channel responsibility. + +The verifier publishes a digest of normalized, path-free trace facts together +with the exact exec counts and zero network/child-process counts. It does not +publish or claim a digest of the temporary raw `strace` files. diff --git a/pyproject.toml b/pyproject.toml index fb0deb1..45653bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,8 +11,23 @@ requires-python = ">=3.11" license = "MIT" license-files = ["LICENSE"] authors = [{name = "Omar Ibrahim"}] +keywords = ["audit", "privacy", "provenance", "security", "supply-chain"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Typing :: Typed", +] dependencies = [] +[project.urls] +Documentation = "https://github.com/omar07ibrahim/Hello-World#readme" +Issues = "https://github.com/omar07ibrahim/Hello-World/issues" +Repository = "https://github.com/omar07ibrahim/Hello-World" + [tool.setuptools] package-dir = {"" = "src"} packages = ["netveil"] diff --git a/tests/test_evidence_renderer.py b/tests/test_evidence_renderer.py new file mode 100644 index 0000000..9448d0a --- /dev/null +++ b/tests/test_evidence_renderer.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import hashlib +import json +import unittest +from xml.etree import ElementTree + +from tools import render_evidence +from tools import verify_fresh_wheel as verifier + +SOURCE_COMMIT = "a" * 40 +SOURCE_DATE_EPOCH = 1_700_000_000 + + +def _receipt() -> dict[str, object]: + report: dict[str, object] = { + "counts": { + "endpoint_occurrences": 5, + "physical_lines": 6, + "source_bytes": len(verifier._CORPUS), + "unique_endpoints": 4, + }, + "duplicates": {"group_count": 1}, + "endpoint_occurrences_by_ip_version": {"ipv4": 4, "ipv6": 1}, + "endpoint_occurrences_by_scope": {"documentation": 5}, + "source_content_id": "nvs1_" + "b" * 64, + } + return { + "report": report, + "report_digest": { + "algorithm": "sha256", + "value": hashlib.sha256( + render_evidence._canonical_json(report) + ).hexdigest(), + }, + "schema": "netveil.aggregate-receipt.v1", + } + + +def _documents() -> tuple[dict[str, object], dict[str, object]]: + artifacts: list[object] = [ + { + "filename": "netveil_audit-0.3.0-py3-none-any.whl", + "kind": "wheel", + "sha256": "c" * 64, + "size_bytes": 24_000, + }, + { + "filename": "netveil_audit-0.3.0.tar.gz", + "kind": "sdist", + "members": [ + { + "kind": "directory", + "mode": "0755", + "path": "netveil_audit-0.3.0", + "size_bytes": 0, + }, + { + "kind": "file", + "mode": "0644", + "path": "netveil_audit-0.3.0/README.md", + "sha256": "d" * 64, + "size_bytes": 100, + }, + ], + "sha256": "e" * 64, + "size_bytes": 31_000, + }, + ] + inventory = { + "artifacts": artifacts, + "schema": render_evidence.INVENTORY_SCHEMA, + "source_commit": SOURCE_COMMIT, + "source_date_epoch": SOURCE_DATE_EPOCH, + } + inventory_payload = render_evidence._json_bytes(inventory) + receipt = _receipt() + receipt_payload = render_evidence._json_bytes(receipt) + verification: dict[str, object] = { + "checks": [{"name": name, "status": "pass"} for name in verifier._CHECKS], + "installed": { + "launcher": { + "logical_path": "bin/netveil-audit", + "mode": "0755", + "sha256": "f" * 64, + "size_bytes": 8_000, + }, + "record": { + "logical_path": ("site-packages/netveil_audit-0.3.0.dist-info/RECORD"), + "mode": "0644", + "sha256": "1" * 64, + "size_bytes": 2_000, + }, + "selected_record_rows": [ + { + "path": "../../../bin/netveil-audit", + "sha256": "f" * 64, + "size_bytes": 8_000, + }, + { + "path": "netveil_audit-0.3.0.dist-info/RECORD", + "sha256": None, + "size_bytes": None, + }, + ], + }, + "integrity_evidence": { + "artifacts": artifacts, + "attestation_verified": False, + "inventory_schema": render_evidence.INVENTORY_SCHEMA, + "inventory_sha256": hashlib.sha256(inventory_payload).hexdigest(), + "inventory_type": "unsigned_sha256_manifest", + "signature_verified": False, + "source_commit": SOURCE_COMMIT, + "source_date_epoch": SOURCE_DATE_EPOCH, + }, + "interpreter": { + "cache_tag": "cpython-312", + "implementation": "cpython", + "version": "3.12.3", + }, + "platform": { + "machine": "x86_64", + "release": "test-kernel", + "sys_platform": "linux", + "system": "Linux", + }, + "public_demo": { + "classification": ( + "synthetic_ietf_documentation_ranges_with_public_demo_key" + ), + "commands": [ + { + "argv": ["netveil-audit", "--version"], + "exit_code": 0, + "stderr": "", + "stdout": "netveil-audit 0.3.0\n", + }, + { + "argv": [ + "netveil-audit", + "receipt", + "documentation-corpus.txt", + "--key-file", + "public-demo.key", + ], + "exit_code": 0, + "stderr": "", + "stdout_json": receipt, + "stdout_sha256": hashlib.sha256(receipt_payload).hexdigest(), + }, + ], + "corpus": { + "physical_lines": 6, + "sha256": hashlib.sha256(verifier._CORPUS).hexdigest(), + "size_bytes": len(verifier._CORPUS), + }, + "public_demo_key": { + "classification": "public_non_secret_test_material", + "sha256": hashlib.sha256(verifier._PUBLIC_DEMO_KEY).hexdigest(), + "size_bytes": 32, + "source_constant": ("tools/verify_fresh_wheel.py:_PUBLIC_DEMO_KEY"), + }, + }, + "schema": render_evidence.VERIFICATION_SCHEMA, + "source_commit": SOURCE_COMMIT, + "status": "pass", + "syscall_traces": [ + { + "exec_chain": ["installed_launcher", "installed_python"], + "exec_count": 2, + "exit_syscall_count": 1, + "label": label, + "network_syscall_count": 0, + "normalized_sha256": digest * 64, + "post_launch_process_count": 0, + "process_count": 1, + } + for label, digest in (("receipt", "2"), ("version", "3")) + ], + "wheel": { + "members": [ + { + "mode": "0644", + "path": "netveil/cli.py", + "sha256": "4" * 64, + "size_bytes": 7_000, + } + ], + "sha256": "c" * 64, + "size_bytes": 24_000, + }, + } + return verification, inventory + + +def _payloads() -> tuple[bytes, bytes]: + verification, inventory = _documents() + return ( + render_evidence._json_bytes(verification), + render_evidence._json_bytes(inventory), + ) + + +class EvidenceRendererTests(unittest.TestCase): + def test_renders_every_visual_from_cross_bound_evidence(self) -> None: + verification, inventory = _payloads() + + outputs = render_evidence.render_bundle( + verification, + inventory, + generator_payload=b"renderer source fixture\n", + ) + + self.assertEqual(set(outputs), set(render_evidence.ALL_OUTPUT_PATHS)) + for path in ( + render_evidence.CLI_SVG_PATH, + render_evidence.COUNTS_SVG_PATH, + render_evidence.MATRIX_SVG_PATH, + render_evidence.PROVENANCE_SVG_PATH, + ): + ElementTree.fromstring(outputs[path]) + self.assertIn(b"captured stdout", outputs[render_evidence.CLI_SVG_PATH]) + self.assertIn(b"endpoint occurrences", outputs[render_evidence.COUNTS_SVG_PATH]) + self.assertIn( + b"coordinated bootstrap record mutation accepted", + outputs[render_evidence.MATRIX_SVG_PATH], + ) + self.assertIn( + SOURCE_COMMIT.encode(), + outputs[render_evidence.PROVENANCE_SVG_PATH], + ) + self.assertIn( + b"netveil-audit receipt documentation-corpus.txt", + outputs[render_evidence.CAST_PATH], + ) + + manifest = json.loads(outputs[render_evidence.MANIFEST_PATH]) + self.assertEqual( + manifest["schema"], + render_evidence.VISUAL_MANIFEST_SCHEMA, + ) + self.assertEqual(manifest["source_commit"], SOURCE_COMMIT) + self.assertEqual( + {record["path"] for record in manifest["outputs"]}, + set(render_evidence.VISUAL_OUTPUT_PATHS), + ) + for record in manifest["outputs"]: + self.assertEqual( + record["sha256"], + hashlib.sha256(outputs[record["path"]]).hexdigest(), + ) + + def test_rendering_is_byte_deterministic(self) -> None: + verification, inventory = _payloads() + + first = render_evidence.render_bundle( + verification, + inventory, + generator_payload=b"same generator\n", + ) + second = render_evidence.render_bundle( + verification, + inventory, + generator_payload=b"same generator\n", + ) + + self.assertEqual(first, second) + + def test_rejects_noncanonical_or_cross_document_drift(self) -> None: + verification, inventory = _documents() + noncanonical = ( + json.dumps(verification, indent=2, sort_keys=True).encode("ascii") + b"\n" + ) + with self.assertRaisesRegex( + render_evidence.EvidenceRenderError, + "not canonical", + ): + render_evidence.parse_evidence( + noncanonical, + render_evidence._json_bytes(inventory), + ) + + verification["source_commit"] = "b" * 40 + with self.assertRaisesRegex( + render_evidence.EvidenceRenderError, + "source commits differ", + ): + render_evidence.parse_evidence( + render_evidence._json_bytes(verification), + render_evidence._json_bytes(inventory), + ) + + def test_rejects_raw_endpoint_text_and_receipt_digest_drift(self) -> None: + verification, inventory = _documents() + verification["unexpected_raw_endpoint"] = "192.0.2.10" + with self.assertRaisesRegex( + render_evidence.EvidenceRenderError, + "raw endpoint", + ): + render_evidence.parse_evidence( + render_evidence._json_bytes(verification), + render_evidence._json_bytes(inventory), + ) + + verification, inventory = _documents() + demo = verification["public_demo"] + assert isinstance(demo, dict) + commands = demo["commands"] + assert isinstance(commands, list) + receipt_command = commands[1] + assert isinstance(receipt_command, dict) + receipt_command["stdout_sha256"] = "0" * 64 + with self.assertRaisesRegex( + render_evidence.EvidenceRenderError, + "receipt stdout digest", + ): + render_evidence.parse_evidence( + render_evidence._json_bytes(verification), + render_evidence._json_bytes(inventory), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fresh_wheel_verifier.py b/tests/test_fresh_wheel_verifier.py new file mode 100644 index 0000000..e3db31a --- /dev/null +++ b/tests/test_fresh_wheel_verifier.py @@ -0,0 +1,924 @@ +from __future__ import annotations + +import csv +import gzip +import hashlib +import importlib.util +import io +import json +import marshal +import stat +import tarfile +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from types import CodeType +from unittest.mock import patch +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +from tools import verify_fresh_wheel as verifier + +_SOURCE_COMMIT = "a" * 40 +_SOURCE_DATE_EPOCH = 1_700_000_000 +_LAUNCHER = ( + b"#!/bin/sh\n" + b'""":"\n' + b'case "$0" in\n' + b" */*) netveil_script_directory=${0%/*} ;;\n" + b" *) exit 70 ;;\n" + b"esac\n" + b'exec "$netveil_script_directory/python" -IESB "$0" "$@"\n' + b"exit 70\n" + b'":"""\n' + b"\n" + b"raise SystemExit(0)\n" +) + + +def _record_payload(members: dict[str, bytes]) -> bytes: + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + record_name = f"{verifier._DIST_INFO}/RECORD" + for name in sorted((*members, record_name)): + if name == record_name: + writer.writerow((name, "", "")) + else: + payload = members[name] + writer.writerow( + ( + name, + f"sha256={verifier._record_digest(payload)}", + str(len(payload)), + ) + ) + return output.getvalue().encode() + + +def _wheel_members() -> dict[str, bytes]: + members = { + verifier._WHEEL_SCRIPT: _LAUNCHER, + "netveil/__init__.py": b"", + "netveil/cli.py": b"", + "netveil/model.py": b"", + "netveil/parser.py": b"", + "netveil/privacy.py": b"", + "netveil/py.typed": b"", + verifier._BOOTSTRAP_NAME: b"", + f"{verifier._DIST_INFO}/METADATA": ( + b"Metadata-Version: 2.4\n" + b"Name: netveil-audit\n" + b"Version: 0.3.0\n" + b"Requires-Python: >=3.11\n" + ), + f"{verifier._DIST_INFO}/WHEEL": ( + b"Wheel-Version: 1.0\n" + b"Generator: verifier-test\n" + b"Root-Is-Purelib: true\n" + b"Tag: py3-none-any\n" + ), + f"{verifier._DIST_INFO}/licenses/LICENSE": b"test-only license\n", + f"{verifier._DIST_INFO}/top_level.txt": (b"netveil\nnetveil_bootstrap\n"), + } + members[f"{verifier._DIST_INFO}/RECORD"] = _record_payload(members) + return members + + +def _zip_bytes(members: dict[str, bytes]) -> bytes: + output = io.BytesIO() + with ZipFile(output, "w") as archive: + for name, payload in members.items(): + info = ZipInfo(name) + info.create_system = 3 + permissions = 0o755 if name == verifier._WHEEL_SCRIPT else 0o644 + info.external_attr = (stat.S_IFREG | permissions) << 16 + info.compress_type = ZIP_DEFLATED + archive.writestr(info, payload) + return output.getvalue() + + +def _installed_record( + *, + site_root: Path, + launcher: Path, + launcher_payload: bytes, +) -> bytes: + relative = verifier._installed_record_path(site_root, launcher) + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + writer.writerow( + ( + relative, + f"sha256={verifier._record_digest(launcher_payload)}", + str(len(launcher_payload)), + ) + ) + writer.writerow((f"{verifier._DIST_INFO}/RECORD", "", "")) + return output.getvalue().encode() + + +def _receipt(key: bytes) -> bytes: + report: dict[str, object] = { + "counts": { + "endpoint_occurrences": 5, + "physical_lines": 6, + "source_bytes": len(verifier._CORPUS), + "unique_endpoints": 4, + }, + "duplicates": {"group_count": 1}, + "endpoint_occurrences_by_ip_version": {"ipv4": 4, "ipv6": 1}, + "endpoint_occurrences_by_scope": {"documentation": 5}, + "source_content_id": "nvs1_" + "0" * 64, + } + document = { + "report": report, + "report_digest": { + "algorithm": "sha256", + "value": hashlib.sha256(verifier._canonical_json(report)).hexdigest(), + }, + "schema": "netveil.aggregate-receipt.v1", + } + payload = verifier._canonical_json(document) + b"\n" + if key in payload: + raise AssertionError("test fixture unexpectedly contains private key") + return payload + + +def _sdist_payload() -> bytes: + tar_payload = io.BytesIO() + with tarfile.open( + fileobj=tar_payload, + mode="w:", + format=tarfile.USTAR_FORMAT, + ) as archive: + directory = tarfile.TarInfo("netveil_audit-0.3.0/") + directory.type = tarfile.DIRTYPE + directory.mode = 0o755 + directory.mtime = _SOURCE_DATE_EPOCH + directory.uid = 0 + directory.gid = 0 + directory.uname = "" + directory.gname = "" + archive.addfile(directory) + + readme_payload = b"# Netveil test sdist\n" + readme = tarfile.TarInfo("netveil_audit-0.3.0/README.md") + readme.type = tarfile.REGTYPE + readme.mode = 0o644 + readme.mtime = _SOURCE_DATE_EPOCH + readme.uid = 0 + readme.gid = 0 + readme.uname = "" + readme.gname = "" + readme.size = len(readme_payload) + archive.addfile(readme, io.BytesIO(readme_payload)) + compressed = io.BytesIO() + with gzip.GzipFile( + filename="", + mode="wb", + compresslevel=9, + fileobj=compressed, + mtime=_SOURCE_DATE_EPOCH, + ) as archive: + archive.write(tar_payload.getvalue()) + return compressed.getvalue() + + +def _sdist_inventory_members() -> list[dict[str, object]]: + readme_payload = b"# Netveil test sdist\n" + return [ + { + "kind": "directory", + "mode": "0755", + "path": "netveil_audit-0.3.0", + "size_bytes": 0, + }, + { + "kind": "file", + "mode": "0644", + "path": "netveil_audit-0.3.0/README.md", + "sha256": hashlib.sha256(readme_payload).hexdigest(), + "size_bytes": len(readme_payload), + }, + ] + + +def _inventory_payload( + wheel_payload: bytes, + *, + source_commit: str = _SOURCE_COMMIT, + wheel_filename: str = verifier._WHEEL_NAME, + wheel_sha256: str | None = None, + wheel_size_bytes: int | None = None, + sdist_sha256: str | None = None, +) -> bytes: + sdist_payload = _sdist_payload() + artifacts = [ + { + "filename": wheel_filename, + "kind": "wheel", + "sha256": ( + hashlib.sha256(wheel_payload).hexdigest() + if wheel_sha256 is None + else wheel_sha256 + ), + "size_bytes": ( + len(wheel_payload) if wheel_size_bytes is None else wheel_size_bytes + ), + }, + { + "filename": verifier._SDIST_NAME, + "kind": "sdist", + "members": _sdist_inventory_members(), + "sha256": ( + hashlib.sha256(sdist_payload).hexdigest() + if sdist_sha256 is None + else sdist_sha256 + ), + "size_bytes": len(sdist_payload), + }, + ] + artifacts.sort(key=lambda artifact: str(artifact["filename"])) + return ( + verifier._canonical_json( + { + "artifacts": artifacts, + "schema": verifier._INVENTORY_SCHEMA, + "source_commit": source_commit, + "source_date_epoch": _SOURCE_DATE_EPOCH, + } + ) + + b"\n" + ) + + +class WheelArchiveContractTests(unittest.TestCase): + def test_accepts_exact_inventory_and_record_bound_launcher(self) -> None: + payload = _zip_bytes(_wheel_members()) + + evidence = verifier._inspect_wheel(payload) + + self.assertEqual(evidence.payload, payload) + self.assertEqual(evidence.launcher, _LAUNCHER) + self.assertEqual(evidence.sha256, hashlib.sha256(payload).hexdigest()) + self.assertEqual( + [member.path for member in evidence.members], + sorted(_wheel_members()), + ) + self.assertTrue( + all(member.mode in ("0644", "0755") for member in evidence.members) + ) + + def test_rejects_entry_points_even_when_recorded(self) -> None: + members = _wheel_members() + record_name = f"{verifier._DIST_INFO}/RECORD" + members.pop(record_name) + members[f"{verifier._DIST_INFO}/entry_points.txt"] = ( + b"[console_scripts]\nnetveil-audit=netveil_bootstrap:entrypoint\n" + ) + members[record_name] = _record_payload(members) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^wheel_inventory_invalid$", + ): + verifier._inspect_wheel(_zip_bytes(members)) + + def test_rejects_stale_record_hash(self) -> None: + members = _wheel_members() + members["netveil/cli.py"] = b"# unrecorded mutation\n" + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^record_mismatch$", + ): + verifier._inspect_wheel(_zip_bytes(members)) + + def test_rejects_duplicate_record_paths(self) -> None: + digest = verifier._record_digest(b"x") + payload = ( + f"netveil/cli.py,sha256={digest},1\nnetveil/cli.py,sha256={digest},1\n" + ).encode() + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^record_invalid$", + ): + verifier._parse_record(payload) + + def test_exact_wheel_reader_rejects_symlink(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + target = root / "target.whl" + target.write_bytes(b"wheel") + link = root / verifier._WHEEL_NAME + link.symlink_to(target) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^wheel_path_invalid$", + ): + verifier._read_exact_wheel(link) + + +class ReleaseInventoryContractTests(unittest.TestCase): + def test_accepts_exact_canonical_unsigned_inventory(self) -> None: + wheel_payload = b"test wheel bytes" + payload = _inventory_payload(wheel_payload) + + inventory = verifier._parse_release_inventory(payload) + + self.assertEqual(inventory.source_commit, _SOURCE_COMMIT) + self.assertEqual(inventory.source_date_epoch, 1_700_000_000) + self.assertEqual(inventory.sha256, hashlib.sha256(payload).hexdigest()) + self.assertEqual( + {artifact.kind for artifact in inventory.artifacts}, + {"wheel", "sdist"}, + ) + + def test_rejects_malformed_and_noncanonical_inventory(self) -> None: + canonical = _inventory_payload(b"test wheel bytes") + document = json.loads(canonical) + malformed_document = dict(document) + malformed_document.pop("source_commit") + malformed = verifier._canonical_json(malformed_document) + b"\n" + noncanonical = json.dumps(document, indent=2).encode("ascii") + b"\n" + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_invalid$", + ): + verifier._parse_release_inventory(malformed) + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_not_canonical$", + ): + verifier._parse_release_inventory(noncanonical) + + def test_rejects_duplicate_inventory_keys(self) -> None: + payload = _inventory_payload(b"test wheel bytes") + duplicated = payload.replace( + b'{"artifacts":', + b'{"schema":"netveil.release-inventory.v1","artifacts":', + 1, + ) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_invalid$", + ): + verifier._parse_release_inventory(duplicated) + + def test_rejects_commit_name_size_and_sha256_mismatches(self) -> None: + wheel_payload = b"not required to be a valid wheel for binding failures" + cases = ( + ( + "commit", + _inventory_payload(wheel_payload), + "b" * 40, + "inventory_source_commit_mismatch", + ), + ( + "name", + _inventory_payload( + wheel_payload, + wheel_filename="unexpected-0.3.0-py3-none-any.whl", + ), + _SOURCE_COMMIT, + "inventory_wheel_name_mismatch", + ), + ( + "size", + _inventory_payload( + wheel_payload, + wheel_size_bytes=len(wheel_payload) + 1, + ), + _SOURCE_COMMIT, + "inventory_wheel_size_mismatch", + ), + ( + "sha256", + _inventory_payload( + wheel_payload, + wheel_sha256="b" * 64, + ), + _SOURCE_COMMIT, + "inventory_wheel_sha256_mismatch", + ), + ( + "sdist_sha256", + _inventory_payload( + wheel_payload, + sdist_sha256="b" * 64, + ), + _SOURCE_COMMIT, + "inventory_sdist_sha256_mismatch", + ), + ) + for label, inventory_payload, expected_commit, failure_code in cases: + with self.subTest(label=label), tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + wheel = root / verifier._WHEEL_NAME + sdist = root / verifier._SDIST_NAME + inventory = root / verifier._INVENTORY_NAME + wheel.write_bytes(wheel_payload) + sdist.write_bytes(_sdist_payload()) + inventory.write_bytes(inventory_payload) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + f"^{failure_code}$", + ): + verifier.verify_wheel( + wheel, + inventory_path=inventory, + sdist_path=sdist, + source_commit=expected_commit, + ) + + def test_sdist_member_inventory_matches_safe_canonical_archive(self) -> None: + payload = _sdist_payload() + inventory = verifier._parse_release_inventory( + _inventory_payload(b"test wheel bytes") + ) + actual = verifier._inspect_sdist( + payload, + source_date_epoch=_SOURCE_DATE_EPOCH, + ) + + verifier._bind_sdist_members(inventory, actual) + self.assertEqual( + [member.path for member in actual], + [ + "netveil_audit-0.3.0", + "netveil_audit-0.3.0/README.md", + ], + ) + + def test_sdist_member_inventory_mismatch_is_rejected(self) -> None: + payload = _inventory_payload(b"test wheel bytes") + document = json.loads(payload) + sdist = next( + artifact + for artifact in document["artifacts"] + if artifact["kind"] == "sdist" + ) + sdist["members"][1]["sha256"] = "b" * 64 + inventory = verifier._parse_release_inventory( + verifier._canonical_json(document) + b"\n" + ) + actual = verifier._inspect_sdist( + _sdist_payload(), + source_date_epoch=_SOURCE_DATE_EPOCH, + ) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_sdist_members_mismatch$", + ): + verifier._bind_sdist_members(inventory, actual) + + def test_windows_drive_like_sdist_members_are_rejected_consistently( + self, + ) -> None: + payload = _inventory_payload(b"test wheel bytes") + document = json.loads(payload) + sdist = next( + artifact + for artifact in document["artifacts"] + if artifact["kind"] == "sdist" + ) + sdist["members"][0]["path"] = "C:escape" + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_invalid$", + ): + verifier._parse_release_inventory( + verifier._canonical_json(document) + b"\n" + ) + + member = tarfile.TarInfo("C:escape") + member.type = tarfile.REGTYPE + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^sdist_member_unsafe$", + ): + verifier._safe_sdist_member_name(member) + + +class InstalledLayoutContractTests(unittest.TestCase): + def create_layout(self, root: Path) -> tuple[Path, Path]: + prefix = root / "venv" + binary = prefix / "bin" + site_root = prefix / "lib" / "python3.11" / "site-packages" + dist_info = site_root / verifier._DIST_INFO + package = site_root / "netveil" + binary.mkdir(parents=True) + dist_info.mkdir(parents=True) + package.mkdir() + python = binary / "python" + python.write_bytes(b"test interpreter placeholder") + launcher = binary / verifier._LAUNCHER_NAME + launcher.write_bytes(_LAUNCHER) + launcher.chmod(0o755) + (site_root / verifier._BOOTSTRAP_NAME).write_bytes(b"") + (package / "cli.py").write_bytes(b"") + (dist_info / "METADATA").write_bytes( + b"Metadata-Version: 2.4\nName: netveil-audit\nVersion: 0.3.0\n" + ) + (dist_info / "RECORD").write_bytes( + _installed_record( + site_root=site_root, + launcher=launcher, + launcher_payload=_LAUNCHER, + ) + ) + return prefix, python + + def test_binds_installed_launcher_to_record_and_metadata(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + prefix, python = self.create_layout(root) + + layout = verifier._inspect_install(prefix, python, _LAUNCHER) + + self.assertEqual(layout.launcher.read_bytes(), _LAUNCHER) + self.assertEqual(layout.dist_info.name, verifier._DIST_INFO) + self.assertIsNotNone(layout.evidence) + assert layout.evidence is not None + self.assertEqual(layout.evidence.launcher.mode, "0755") + self.assertEqual( + layout.evidence.launcher.sha256, + hashlib.sha256(_LAUNCHER).hexdigest(), + ) + self.assertGreater(layout.evidence.record.size_bytes, 0) + self.assertEqual( + {row.path for row in layout.evidence.selected_record_rows}, + { + verifier._installed_record_path( + layout.site_root, + layout.launcher, + ), + f"{verifier._DIST_INFO}/RECORD", + }, + ) + + def test_rejects_installed_entry_points(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + prefix, python = self.create_layout(root) + dist_info = next(prefix.glob("lib/python*/site-packages/*.dist-info")) + (dist_info / "entry_points.txt").write_bytes( + b"[console_scripts]\nnetveil-audit=netveil_bootstrap:entrypoint\n" + ) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^entry_points_present$", + ): + verifier._inspect_install(prefix, python, _LAUNCHER) + + +class AdversarialPrimitiveTests(unittest.TestCase): + def test_record_rewrite_changes_only_selected_binding(self) -> None: + original = ( + b"netveil_bootstrap.py,sha256=" + + verifier._record_digest(b"old").encode() + + b",3\nother.py,sha256=" + + verifier._record_digest(b"same").encode() + + b",4\n" + ) + + updated = verifier._updated_record( + original, + path="netveil_bootstrap.py", + replacement=b"new bytes", + ) + records = verifier._parse_record(updated) + + self.assertEqual( + records["netveil_bootstrap.py"].digest, + verifier._record_digest(b"new bytes"), + ) + self.assertEqual( + records["other.py"].digest, + verifier._record_digest(b"same"), + ) + + def test_temporary_mutation_restores_bytes_and_mode_after_failure(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + path = Path(raw_root) / "artifact.py" + path.write_bytes(b"original") + path.chmod(0o640) + + with ( + self.assertRaisesRegex(RuntimeError, "^stop$"), + verifier._temporary_bytes(path, b"tampered"), + ): + self.assertEqual(path.read_bytes(), b"tampered") + raise RuntimeError("stop") + + self.assertEqual(path.read_bytes(), b"original") + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o640) + + def test_unchecked_hash_pyc_contains_adversarial_code_object(self) -> None: + payload = verifier._unchecked_hash_pyc( + b"sentinel = 7\n", + b"trusted source\n", + ) + + self.assertEqual(payload[:4], importlib.util.MAGIC_NUMBER) + self.assertEqual(int.from_bytes(payload[4:8], "little"), 1) + code = marshal.loads(payload[16:]) + self.assertIsInstance(code, CodeType) + namespace: dict[str, object] = {} + exec(code, namespace) # noqa: S102 - executes fixed unit-test bytes. + self.assertEqual(namespace["sentinel"], 7) + + def test_receipt_validator_accepts_canonical_redacted_fixture(self) -> None: + key = b"k" * 32 + + document = verifier._verify_receipt_document(_receipt(key), key=key) + + self.assertEqual(document["schema"], "netveil.aggregate-receipt.v1") + + def test_public_demo_key_is_explicit_non_secret_test_material(self) -> None: + self.assertEqual(len(verifier._PUBLIC_DEMO_KEY), 32) + self.assertEqual( + hashlib.sha256(verifier._PUBLIC_DEMO_KEY).hexdigest(), + "2d27befbc438954c4a55d8c0e36192c5a4a7e9e3f15c17d6e427c4a3499d945d", + ) + + def test_receipt_validator_rejects_private_key_material(self) -> None: + key = b"k" * 32 + payload = _receipt(key) + document = json.loads(payload) + document["private"] = key.hex() + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^receipt_private_data_detected$", + ): + verifier._verify_receipt_document( + verifier._canonical_json(document) + b"\n", + key=key, + ) + + def test_syscall_trace_accepts_only_launcher_python_exec_chain(self) -> None: + root = Path("/private-verifier-root") + layout = verifier.InstalledLayout( + prefix=root / "venv", + python=root / "venv/bin/python", + launcher=root / "venv/bin/netveil-audit", + site_root=root / "venv/lib/python3.11/site-packages", + dist_info=root / "dist-info", + bootstrap=root / "bootstrap.py", + package_root=root / "netveil", + record=root / "RECORD", + ) + payload = ( + f'execve("{layout.launcher}", ["netveil-audit"], 0x0) = 0\n' + f'execve("{layout.python}", ["python", "-IESB"], 0x0) = 0\n' + "exit_group(0) = ?\n" + ).encode() + + evidence = verifier._validate_syscall_trace( + (payload,), + layout=layout, + label="version", + ) + + self.assertEqual( + evidence.exec_chain, + ("installed_launcher", "installed_python"), + ) + self.assertEqual(evidence.network_syscall_count, 0) + self.assertEqual(evidence.post_launch_process_count, 0) + self.assertEqual(len(evidence.normalized_sha256), 64) + + def test_syscall_trace_rejects_network_and_post_launch_processes(self) -> None: + root = Path("/private-verifier-root") + layout = verifier.InstalledLayout( + prefix=root / "venv", + python=root / "venv/bin/python", + launcher=root / "venv/bin/netveil-audit", + site_root=root / "venv/lib/python3.11/site-packages", + dist_info=root / "dist-info", + bootstrap=root / "bootstrap.py", + package_root=root / "netveil", + record=root / "RECORD", + ) + prefix = ( + f'execve("{layout.launcher}", ["netveil-audit"], 0x0) = 0\n' + f'execve("{layout.python}", ["python", "-IESB"], 0x0) = 0\n' + ) + for syscall in ( + "socket(AF_INET, SOCK_STREAM, IPPROTO_IP) = 3\n", + "clone(child_stack=NULL, flags=SIGCHLD) = 42\n", + ): + with ( + self.subTest(syscall=syscall), + self.assertRaisesRegex( + verifier.VerificationFailure, + "^network_or_process_activity_detected$", + ), + ): + verifier._validate_syscall_trace( + ((prefix + syscall + "exit_group(0) = ?\n").encode(),), + layout=layout, + ) + + def test_source_commit_must_be_exact_lowercase_sha1(self) -> None: + verifier._validate_source_commit(_SOURCE_COMMIT) + for invalid in ("a" * 39, "A" * 40, "g" * 40, "/private/path"): + with ( + self.subTest(invalid=invalid), + self.assertRaisesRegex( + verifier.VerificationFailure, + "^source_commit_invalid$", + ), + ): + verifier._validate_source_commit(invalid) + + +class VerifierCliTests(unittest.TestCase): + def test_success_json_contains_no_input_path(self) -> None: + stdout = io.StringIO() + stderr = io.StringIO() + summary = verifier.VerificationSummary( + source_commit=_SOURCE_COMMIT, + release_inventory=verifier._parse_release_inventory( + _inventory_payload(b"summary wheel bytes") + ), + installed=verifier.InstalledEvidence( + launcher=verifier.InstalledFileEvidence( + logical_path="bin/netveil-audit", + mode="0755", + sha256="c" * 64, + size_bytes=456, + ), + record=verifier.InstalledFileEvidence( + logical_path=("site-packages/netveil_audit-0.3.0.dist-info/RECORD"), + mode="0644", + sha256="d" * 64, + size_bytes=789, + ), + selected_record_rows=( + verifier.InstalledRecordRowEvidence( + path="../../../bin/netveil-audit", + sha256="c" * 64, + size_bytes=456, + ), + ), + ), + interpreter=verifier.InterpreterEvidence( + implementation="cpython", + version="3.11.0", + cache_tag="cpython-311", + ), + platform=verifier.PlatformEvidence( + sys_platform="linux", + system="Linux", + release="test-kernel", + machine="x86_64", + ), + syscall_traces=( + verifier.TraceEvidence( + label="version", + normalized_sha256="e" * 64, + process_count=1, + exec_chain=("installed_launcher", "installed_python"), + exec_count=2, + exit_syscall_count=1, + network_syscall_count=0, + post_launch_process_count=0, + ), + ), + public_demo=verifier.PublicDemoEvidence( + corpus_sha256=hashlib.sha256(verifier._CORPUS).hexdigest(), + corpus_size_bytes=len(verifier._CORPUS), + corpus_physical_lines=verifier._CORPUS.count(b"\n"), + public_key_sha256=hashlib.sha256(verifier._PUBLIC_DEMO_KEY).hexdigest(), + public_key_size_bytes=len(verifier._PUBLIC_DEMO_KEY), + version_stdout="netveil-audit 0.3.0\n", + receipt=json.loads(_receipt(verifier._PUBLIC_DEMO_KEY)), + receipt_stdout_sha256=hashlib.sha256( + _receipt(verifier._PUBLIC_DEMO_KEY) + ).hexdigest(), + ), + wheel_sha256="a" * 64, + wheel_size_bytes=123, + wheel_members=( + verifier.WheelMemberEvidence( + path="netveil/cli.py", + sha256="b" * 64, + size=7, + mode="0644", + ), + ), + ) + + with ( + patch.object(verifier, "verify_wheel", return_value=summary), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + result = verifier.main( + [ + "--source-commit", + _SOURCE_COMMIT, + "--inventory", + "/home/private/workspace/release-inventory.json", + "--sdist", + "/home/private/workspace/netveil_audit-0.3.0.tar.gz", + ("/home/private/workspace/netveil_audit-0.3.0-py3-none-any.whl"), + ] + ) + + self.assertEqual(result, 0) + self.assertEqual(stderr.getvalue(), "") + document = json.loads(stdout.getvalue()) + self.assertEqual(document["status"], "pass") + self.assertEqual(document["source_commit"], _SOURCE_COMMIT) + self.assertEqual( + document["integrity_evidence"]["inventory_type"], + "unsigned_sha256_manifest", + ) + self.assertFalse(document["integrity_evidence"]["signature_verified"]) + self.assertFalse(document["integrity_evidence"]["attestation_verified"]) + self.assertEqual(document["installed"]["launcher"]["mode"], "0755") + self.assertEqual(document["installed"]["record"]["size_bytes"], 789) + self.assertEqual(document["platform"]["machine"], "x86_64") + self.assertEqual( + document["public_demo"]["classification"], + "synthetic_ietf_documentation_ranges_with_public_demo_key", + ) + self.assertEqual( + document["public_demo"]["commands"][0]["stdout"], + "netveil-audit 0.3.0\n", + ) + self.assertEqual( + document["public_demo"]["public_demo_key"]["classification"], + "public_non_secret_test_material", + ) + self.assertEqual( + document["syscall_traces"][0]["exec_chain"], + ["installed_launcher", "installed_python"], + ) + self.assertEqual(document["interpreter"]["implementation"], "cpython") + self.assertEqual(document["wheel"]["members"][0]["path"], "netveil/cli.py") + self.assertNotIn("/home/private", stdout.getvalue()) + self.assertEqual( + [item["name"] for item in document["checks"]], + list(verifier._CHECKS), + ) + + def test_failure_is_stable_and_redacts_input_path(self) -> None: + stdout = io.StringIO() + stderr = io.StringIO() + + with ( + patch.object( + verifier, + "verify_wheel", + side_effect=verifier.VerificationFailure("wheel_path_invalid"), + ), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + result = verifier.main( + [ + "--source-commit", + _SOURCE_COMMIT, + "--inventory", + "/home/private/release-inventory.json", + "--sdist", + "/home/private/netveil_audit-0.3.0.tar.gz", + "/home/private/secret.whl", + ] + ) + + self.assertEqual(result, 1) + self.assertEqual(stdout.getvalue(), "") + self.assertEqual( + stderr.getvalue(), + "netveil-wheel-verifier: wheel_path_invalid\n", + ) + self.assertNotIn("/home/private", stderr.getvalue()) + + def test_usage_is_stable(self) -> None: + stderr = io.StringIO() + + with redirect_stderr(stderr): + result = verifier.main([]) + + self.assertEqual(result, 2) + self.assertEqual( + stderr.getvalue(), + "netveil-wheel-verifier: usage_error\n", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 1001941..7ad3b72 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -48,6 +48,17 @@ def test_developer_tool_versions_are_exactly_pinned(self) -> None: self.assertGreaterEqual(len(pins), 4) self.assertTrue(all(line.count("==") == 1 for line in pins)) + def test_package_metadata_identifies_omar_and_public_project_links(self) -> None: + document = tomllib.loads((ROOT / "pyproject.toml").read_text()) + project = document["project"] + + self.assertEqual(project["authors"], [{"name": "Omar Ibrahim"}]) + self.assertEqual( + project["urls"]["Repository"], + "https://github.com/omar07ibrahim/Hello-World", + ) + self.assertIn("Typing :: Typed", project["classifiers"]) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_release_builder.py b/tests/test_release_builder.py new file mode 100644 index 0000000..69a017b --- /dev/null +++ b/tests/test_release_builder.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path +from typing import Any +from unittest.mock import patch + +from tools import build_release + +EPOCH = 1_700_000_000 +SOURCE_COMMIT = "a" * 40 +ROOT = Path(__file__).resolve().parents[1] + + +def _gzip_tar( + members: tuple[tuple[str, bytes | None, int, bytes], ...], + *, + gzip_filename: str, + gzip_mtime: int, + reverse: bool = False, +) -> bytes: + tar_payload = io.BytesIO() + ordered = tuple(reversed(members)) if reverse else members + with tarfile.open( + fileobj=tar_payload, + mode="w:", + format=tarfile.PAX_FORMAT, + ) as archive: + for index, (name, payload, mode, member_type) in enumerate(ordered): + information = tarfile.TarInfo(name) + information.type = member_type + information.mode = mode + information.mtime = 111 + index + information.uid = 1000 + index + information.gid = 2000 + index + information.uname = f"user-{index}" + information.gname = f"group-{index}" + information.size = 0 if payload is None else len(payload) + information.pax_headers = {"comment": f"nondeterministic-{index}"} + archive.addfile( + information, + None if payload is None else io.BytesIO(payload), + ) + compressed = io.BytesIO() + with gzip.GzipFile( + filename=gzip_filename, + mode="wb", + compresslevel=1 if reverse else 6, + fileobj=compressed, + mtime=gzip_mtime, + ) as archive: + archive.write(tar_payload.getvalue()) + return compressed.getvalue() + + +def _safe_members() -> tuple[tuple[str, bytes | None, int, bytes], ...]: + return ( + ("netveil_audit-0.3.0/", None, 0o700, tarfile.DIRTYPE), + ( + "netveil_audit-0.3.0/README.md", + b"# Netveil\n", + 0o600, + tarfile.REGTYPE, + ), + ( + "netveil_audit-0.3.0/scripts/", + None, + 0o777, + tarfile.DIRTYPE, + ), + ( + "netveil_audit-0.3.0/scripts/netveil-audit", + b"#!/bin/sh\nexit 0\n", + 0o700, + tarfile.REGTYPE, + ), + ) + + +def _git(project: Path, *arguments: str) -> subprocess.CompletedProcess[bytes]: + environment = dict(os.environ) + environment.update( + { + "GIT_AUTHOR_EMAIL": "31526072+omar07ibrahim@users.noreply.github.com", + "GIT_AUTHOR_NAME": "Omar Ibrahim", + "GIT_COMMITTER_EMAIL": ("31526072+omar07ibrahim@users.noreply.github.com"), + "GIT_COMMITTER_NAME": "Omar Ibrahim", + } + ) + return subprocess.run( + ("git", *arguments), + cwd=project, + env=environment, + stdin=subprocess.DEVNULL, + capture_output=True, + check=True, + ) + + +def _initialize_repository(project: Path) -> str: + project.mkdir() + (project / ".gitignore").write_text("ignored.generated\n") + (project / "tracked.txt").write_text("tracked commit bytes\n") + _git(project, "init", "--quiet") + _git(project, "add", ".gitignore", "tracked.txt") + _git(project, "commit", "--quiet", "-m", "Create test fixture") + return _git(project, "rev-parse", "HEAD").stdout.decode("ascii").strip() + + +class ReleaseBuilderTests(unittest.TestCase): + def test_logically_identical_sdists_normalize_byte_identically(self) -> None: + first = _gzip_tar( + _safe_members(), + gzip_filename="first-random-name.tar", + gzip_mtime=123, + ) + second = _gzip_tar( + _safe_members(), + gzip_filename="second-random-name.tar", + gzip_mtime=987_654, + reverse=True, + ) + + normalized_first = build_release.normalize_sdist_bytes( + first, + source_date_epoch=EPOCH, + ) + normalized_second = build_release.normalize_sdist_bytes( + second, + source_date_epoch=EPOCH, + ) + self.assertEqual(normalized_first, normalized_second) + self.assertEqual(normalized_first[3], 0) + self.assertEqual( + int.from_bytes(normalized_first[4:8], "little"), + EPOCH, + ) + + with tarfile.open( + fileobj=io.BytesIO(normalized_first), + mode="r:gz", + ) as archive: + members = archive.getmembers() + self.assertEqual( + [member.name for member in members], + [ + "netveil_audit-0.3.0", + "netveil_audit-0.3.0/README.md", + "netveil_audit-0.3.0/scripts", + "netveil_audit-0.3.0/scripts/netveil-audit", + ], + ) + for member in members: + self.assertEqual(member.mtime, EPOCH) + self.assertEqual(member.uid, 0) + self.assertEqual(member.gid, 0) + self.assertEqual(member.uname, "") + self.assertEqual(member.gname, "") + self.assertEqual( + [member.mode for member in members], + [0o755, 0o644, 0o755, 0o755], + ) + + def test_unsafe_archive_members_are_rejected(self) -> None: + cases = { + "absolute": (("/absolute.txt", b"x", 0o644, tarfile.REGTYPE),), + "backslash": (("root\\escape.txt", b"x", 0o644, tarfile.REGTYPE),), + "character_device": (("root/device", None, 0o600, tarfile.CHRTYPE),), + "duplicate": ( + ("root/file.txt", b"one", 0o644, tarfile.REGTYPE), + ("root/file.txt", b"two", 0o644, tarfile.REGTYPE), + ), + "hard_link": (("root/link", None, 0o644, tarfile.LNKTYPE),), + "symlink": (("root/link", None, 0o777, tarfile.SYMTYPE),), + "traversal": (("root/../../escape.txt", b"x", 0o644, tarfile.REGTYPE),), + } + for label, members in cases.items(): + with self.subTest(label=label): + payload = _gzip_tar( + members, + gzip_filename=f"{label}.tar", + gzip_mtime=1, + ) + with self.assertRaises(build_release.ReleaseBuildError): + build_release.normalize_sdist_bytes( + payload, + source_date_epoch=EPOCH, + ) + + def test_inventory_is_stable_sorted_and_hash_bound(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + wheel = root / "netveil_audit-0.3.0-py3-none-any.whl" + sdist = root / "netveil_audit-0.3.0.tar.gz" + wheel.write_bytes(b"backend-owned-wheel-bytes") + sdist.write_bytes( + build_release.normalize_sdist_bytes( + _gzip_tar( + _safe_members(), + gzip_filename="source.tar", + gzip_mtime=99, + ), + source_date_epoch=EPOCH, + ) + ) + first = build_release.artifact_inventory( + (wheel, sdist), + source_date_epoch=EPOCH, + source_commit=SOURCE_COMMIT, + ) + second = build_release.artifact_inventory( + (sdist, wheel), + source_date_epoch=EPOCH, + source_commit=SOURCE_COMMIT, + ) + + self.assertEqual(first, second) + document: dict[str, Any] = json.loads(first) + self.assertEqual(document["schema"], build_release.INVENTORY_SCHEMA) + self.assertEqual(document["source_commit"], SOURCE_COMMIT) + self.assertEqual(document["source_date_epoch"], EPOCH) + artifacts = document["artifacts"] + self.assertEqual( + [artifact["filename"] for artifact in artifacts], + sorted((sdist.name, wheel.name)), + ) + by_name = {artifact["filename"]: artifact for artifact in artifacts} + self.assertEqual( + by_name[wheel.name]["sha256"], + hashlib.sha256(b"backend-owned-wheel-bytes").hexdigest(), + ) + self.assertEqual( + by_name[wheel.name]["size_bytes"], + len(b"backend-owned-wheel-bytes"), + ) + sdist_members = by_name[sdist.name]["members"] + self.assertEqual( + [member["path"] for member in sdist_members], + [ + "netveil_audit-0.3.0", + "netveil_audit-0.3.0/README.md", + "netveil_audit-0.3.0/scripts", + "netveil_audit-0.3.0/scripts/netveil-audit", + ], + ) + self.assertNotIn("sha256", sdist_members[0]) + self.assertEqual( + sdist_members[1]["sha256"], + hashlib.sha256(b"# Netveil\n").hexdigest(), + ) + + def test_normalization_never_overwrites_an_existing_destination(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "input" / "netveil_audit-0.3.0.tar.gz" + destination = root / "output" / source.name + source.parent.mkdir() + destination.parent.mkdir() + source.write_bytes( + _gzip_tar( + _safe_members(), + gzip_filename="source.tar", + gzip_mtime=99, + ) + ) + destination.write_bytes(b"retained-existing-artifact") + + with self.assertRaises(build_release.ReleaseBuildError): + build_release.normalize_sdist( + source, + destination, + source_date_epoch=EPOCH, + ) + self.assertEqual( + destination.read_bytes(), + b"retained-existing-artifact", + ) + + def test_sanitized_environment_is_complete_and_reproducible(self) -> None: + home = Path("/private/netveil-build-home") + first = build_release.sanitized_build_environment( + source_date_epoch=EPOCH, + home=home, + ) + second = build_release.sanitized_build_environment( + source_date_epoch=EPOCH, + home=home, + ) + self.assertEqual(first, second) + self.assertEqual(first["SOURCE_DATE_EPOCH"], str(EPOCH)) + self.assertEqual(first["HOME"], str(home)) + self.assertEqual(first["PYTHONHASHSEED"], "0") + for excluded in ( + "GIT_CONFIG_GLOBAL", + "LD_PRELOAD", + "PYTHONHOME", + "PYTHONPATH", + ): + self.assertNotIn(excluded, first) + + def test_pinned_backend_accepts_the_live_project_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "project" + project.mkdir() + for name in ("LICENSE", "README.md", "pyproject.toml"): + shutil.copy2(ROOT / name, project / name) + for name in ("scripts", "src"): + shutil.copytree(ROOT / name, project / name) + home = Path(temporary) / "home" + home.mkdir() + + completed = subprocess.run( + ( + sys.executable, + "-c", + ( + "from setuptools import build_meta;" + "build_meta.get_requires_for_build_sdist()" + ), + ), + cwd=project, + env=dict( + build_release.sanitized_build_environment( + source_date_epoch=EPOCH, + home=home, + ) + ), + stdin=subprocess.DEVNULL, + capture_output=True, + check=False, + timeout=30, + ) + + self.assertEqual( + completed.returncode, + 0, + msg=(completed.stdout + completed.stderr).decode( + "utf-8", + errors="replace", + ), + ) + + def test_git_source_commit_rejects_tracked_and_untracked_changes(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + project = root / "project" + expected_commit = _initialize_repository(project) + home = root / "home" + home.mkdir() + + self.assertEqual( + build_release._git_source_commit(project.resolve(), home), + expected_commit, + ) + tracked = project / "tracked.txt" + tracked.write_text("dirty tracked bytes\n") + with self.assertRaisesRegex( + build_release.ReleaseBuildError, + "^netveil_release_error:worktree_dirty$", + ): + build_release._git_source_commit(project.resolve(), home) + tracked.write_text("tracked commit bytes\n") + + untracked = project / "untracked.txt" + untracked.write_text("dirty untracked bytes\n") + with self.assertRaisesRegex( + build_release.ReleaseBuildError, + "^netveil_release_error:worktree_dirty$", + ): + build_release._git_source_commit(project.resolve(), home) + + def test_exported_source_is_exact_head_and_excludes_ignored_state(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + project = root / "project" + expected_commit = _initialize_repository(project) + (project / "ignored.generated").write_text( + "must not influence the backend\n" + ) + home = root / "home" + home.mkdir() + snapshot = root / "snapshot" + + observed_commit = build_release._export_clean_head( + project.resolve(), + snapshot, + root / "source.tar", + home, + ) + + self.assertEqual(observed_commit, expected_commit) + self.assertEqual( + (snapshot / "tracked.txt").read_text(), + "tracked commit bytes\n", + ) + self.assertEqual( + (snapshot / ".gitignore").read_text(), + "ignored.generated\n", + ) + self.assertFalse((snapshot / "ignored.generated").exists()) + self.assertFalse((snapshot / ".git").exists()) + + def test_release_destination_must_be_outside_source_tree(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "project" + project.mkdir() + + with self.assertRaisesRegex( + build_release.ReleaseBuildError, + "^netveil_release_error:build_path_invalid$", + ): + build_release.build_release( + project, + project / "release", + source_date_epoch=EPOCH, + python_executable=Path(sys.executable), + ) + + def test_build_python_preserves_a_virtual_environment_symlink(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + binary = root / "venv/bin" + binary.mkdir(parents=True) + launcher = binary / "python" + launcher.symlink_to(Path(sys.executable)) + + observed = build_release._build_python_path(launcher) + dereferenced = launcher.resolve() + + self.assertEqual(observed, launcher.absolute()) + self.assertNotEqual(observed, dereferenced) + + def test_backend_orchestration_is_mocked_and_wheel_bytes_are_untouched( + self, + ) -> None: + backend_wheel = b"opaque-backend-wheel-bytes" + backend_sdist = _gzip_tar( + _safe_members(), + gzip_filename="backend-random-name.tar", + gzip_mtime=42, + ) + observed_commands: list[tuple[str, ...]] = [] + observed_environments: list[dict[str, str]] = [] + observed_working_directories: list[Path] = [] + + def fake_export( + project: Path, + destination: Path, + archive_path: Path, + home: Path, + ) -> str: + del project, archive_path, home + destination.mkdir() + return SOURCE_COMMIT + + def fake_run( + command: tuple[str, ...], + *, + cwd: Path, + env: dict[str, str], + stdin: int, + capture_output: bool, + check: bool, + timeout: int, + ) -> subprocess.CompletedProcess[bytes]: + del stdin, capture_output, check, timeout + observed_commands.append(command) + observed_environments.append(env) + observed_working_directories.append(cwd) + backend = Path(command[command.index("--outdir") + 1]) + (backend / "netveil_audit-0.3.0-py3-none-any.whl").write_bytes( + backend_wheel + ) + (backend / "netveil_audit-0.3.0.tar.gz").write_bytes(backend_sdist) + return subprocess.CompletedProcess( + command, + 0, + stdout=b"", + stderr=b"", + ) + + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "project" + project.mkdir() + output = Path(temporary) / "release" + with ( + patch( + "tools.build_release._export_clean_head", + side_effect=fake_export, + ), + patch( + "tools.build_release._git_source_commit", + return_value=SOURCE_COMMIT, + ), + patch( + "tools.build_release.subprocess.run", + side_effect=fake_run, + ), + ): + inventory = build_release.build_release( + project, + output, + source_date_epoch=EPOCH, + python_executable=Path(sys.executable), + ) + + wheel = output / "netveil_audit-0.3.0-py3-none-any.whl" + normalized_sdist = output / "netveil_audit-0.3.0.tar.gz" + self.assertEqual(wheel.read_bytes(), backend_wheel) + self.assertEqual( + normalized_sdist.read_bytes(), + build_release.normalize_sdist_bytes( + backend_sdist, + source_date_epoch=EPOCH, + ), + ) + self.assertEqual( + (output / build_release.INVENTORY_FILENAME).read_bytes(), + inventory, + ) + self.assertEqual(json.loads(inventory)["source_commit"], SOURCE_COMMIT) + + self.assertEqual(len(observed_commands), 1) + command = observed_commands[0] + self.assertEqual( + command[:4], + ( + os.path.abspath(sys.executable), + "-m", + "build", + "--no-isolation", + ), + ) + backend_path = Path(command[command.index("--outdir") + 1]) + self.assertFalse(backend_path.is_relative_to(project)) + self.assertEqual(observed_working_directories, [Path(command[-1])]) + self.assertFalse(observed_working_directories[0].is_relative_to(project)) + self.assertEqual( + observed_environments[0], + dict( + build_release.sanitized_build_environment( + source_date_epoch=EPOCH, + home=Path(observed_environments[0]["HOME"]), + ) + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/build_release.py b/tools/build_release.py new file mode 100755 index 0000000..0ff0aff --- /dev/null +++ b/tools/build_release.py @@ -0,0 +1,821 @@ +#!/usr/bin/env python3 +"""Build and normalize a deterministic Netveil release with the standard library. + +The build backend owns wheel bytes: this helper copies the wheel byte-for-byte +and never opens or repacks it. The setuptools sdist is treated as an untrusted +tar.gz container and rewritten into one canonical archive before publication. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import zlib +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, NoReturn + +INVENTORY_SCHEMA: Final = "netveil.release-inventory.v1" +INVENTORY_FILENAME: Final = "release-inventory.json" + +_MAX_COMPRESSED_SDIST_BYTES: Final = 128 * 1_048_576 +_MAX_TAR_BYTES: Final = 256 * 1_048_576 +_MAX_MEMBER_BYTES: Final = 64 * 1_048_576 +_MAX_TOTAL_FILE_BYTES: Final = 192 * 1_048_576 +_MAX_MEMBERS: Final = 20_000 +_MAX_ARTIFACT_BYTES: Final = 512 * 1_048_576 +_BUILD_TIMEOUT_SECONDS: Final = 10 * 60 +_GIT_TIMEOUT_SECONDS: Final = 60 +_MAX_GIT_OUTPUT_BYTES: Final = 1_048_576 +_TAR_BLOCK_BYTES: Final = 512 +_TAR_TRAILER_BYTES: Final = 2 * _TAR_BLOCK_BYTES +_GZIP_MAX_MTIME: Final = (1 << 32) - 1 +_LOWER_HEX: Final = frozenset("0123456789abcdef") + + +class ReleaseBuildError(RuntimeError): + """A stable release failure that does not disclose archive contents.""" + + __slots__ = ("code",) + + code: str + + def __init__(self, code: str) -> None: + self.code = code + super().__init__(f"netveil_release_error:{code}") + + +@dataclass(frozen=True, slots=True) +class _CanonicalMember: + """One validated logical member ready for canonical tar serialization.""" + + name: str + payload: bytes | None + executable: bool + + @property + def is_directory(self) -> bool: + return self.payload is None + + +def _fail(code: str) -> NoReturn: + raise ReleaseBuildError(code) + + +def _validated_epoch(source_date_epoch: int) -> int: + if ( + type(source_date_epoch) is not int + or not 0 <= source_date_epoch <= _GZIP_MAX_MTIME + ): + _fail("source_date_epoch_invalid") + return source_date_epoch + + +def _validated_source_commit(source_commit: str) -> str: + if ( + type(source_commit) is not str + or len(source_commit) != 40 + or any(character not in _LOWER_HEX for character in source_commit) + ): + _fail("source_commit_invalid") + return source_commit + + +def _canonical_member_name(member: tarfile.TarInfo) -> str: + raw_name = member.name + if ( + type(raw_name) is not str + or not raw_name + or "\\" in raw_name + or "\x00" in raw_name + or any(ord(character) < 32 or ord(character) == 127 for character in raw_name) + ): + _fail("sdist_member_unsafe") + normalized = ( + raw_name[:-1] if member.isdir() and raw_name.endswith("/") else raw_name + ) + path = PurePosixPath(normalized) + first = path.parts[0] if path.parts else "" + if ( + not normalized + or path.is_absolute() + or path.as_posix() != normalized + or any(part in ("", ".", "..") for part in path.parts) + or (len(first) >= 2 and first[0].isalpha() and first[1] == ":") + or (not member.isdir() and raw_name.endswith("/")) + ): + _fail("sdist_member_unsafe") + return normalized + + +def _decompress_sdist(payload: bytes) -> bytes: + if ( + type(payload) is not bytes + or not 0 < len(payload) <= _MAX_COMPRESSED_SDIST_BYTES + ): + _fail("sdist_invalid") + source = io.BytesIO(payload) + try: + with gzip.GzipFile(fileobj=source, mode="rb") as compressed: + tar_payload = compressed.read(_MAX_TAR_BYTES + 1) + except (EOFError, OSError, gzip.BadGzipFile, zlib.error): + _fail("sdist_invalid") + if ( + not 0 < len(tar_payload) <= _MAX_TAR_BYTES + or len(tar_payload) % _TAR_BLOCK_BYTES + or len(tar_payload) < _TAR_TRAILER_BYTES + ): + _fail("sdist_invalid") + return tar_payload + + +def _read_members(tar_payload: bytes) -> tuple[_CanonicalMember, ...]: + observed: dict[str, _CanonicalMember] = {} + total_file_bytes = 0 + archive_offset = -1 + try: + with tarfile.open(fileobj=io.BytesIO(tar_payload), mode="r:") as archive: + members = archive.getmembers() + archive_offset = archive.offset + if not 1 <= len(members) <= _MAX_MEMBERS: + _fail("sdist_invalid") + for member in members: + if not (member.isdir() or member.isreg()): + _fail("sdist_member_unsafe") + if member.sparse is not None: + _fail("sdist_member_unsafe") + name = _canonical_member_name(member) + if name in observed: + _fail("sdist_member_duplicate") + if member.isdir(): + if member.size != 0: + _fail("sdist_member_unsafe") + observed[name] = _CanonicalMember( + name=name, + payload=None, + executable=True, + ) + continue + if not 0 <= member.size <= _MAX_MEMBER_BYTES: + _fail("sdist_member_unsafe") + total_file_bytes += member.size + if total_file_bytes > _MAX_TOTAL_FILE_BYTES: + _fail("sdist_invalid") + extracted = archive.extractfile(member) + if extracted is None: + _fail("sdist_invalid") + content = extracted.read(_MAX_MEMBER_BYTES + 1) + if len(content) != member.size: + _fail("sdist_invalid") + observed[name] = _CanonicalMember( + name=name, + payload=content, + executable=bool(member.mode & 0o111), + ) + except ReleaseBuildError: + raise + except (OSError, tarfile.TarError, UnicodeError, ValueError): + _fail("sdist_invalid") + + if ( + archive_offset < 0 + or len(tar_payload) - archive_offset < _TAR_TRAILER_BYTES + or any(tar_payload[archive_offset:]) + ): + _fail("sdist_invalid") + return tuple(observed[name] for name in sorted(observed)) + + +def _serialize_tar( + members: tuple[_CanonicalMember, ...], + *, + source_date_epoch: int, +) -> bytes: + destination = io.BytesIO() + try: + with tarfile.open( + fileobj=destination, + mode="w:", + format=tarfile.USTAR_FORMAT, + ) as archive: + for member in members: + information = tarfile.TarInfo( + member.name + "/" if member.is_directory else member.name + ) + information.type = ( + tarfile.DIRTYPE if member.is_directory else tarfile.REGTYPE + ) + information.size = 0 if member.payload is None else len(member.payload) + information.mode = ( + 0o755 if member.is_directory or member.executable else 0o644 + ) + information.mtime = source_date_epoch + information.uid = 0 + information.gid = 0 + information.uname = "" + information.gname = "" + information.linkname = "" + information.pax_headers = {} + archive.addfile( + information, + None if member.payload is None else io.BytesIO(member.payload), + ) + except (OSError, tarfile.TarError, UnicodeError, ValueError): + _fail("sdist_not_canonicalizable") + return destination.getvalue() + + +def normalize_sdist_bytes( + payload: bytes, + *, + source_date_epoch: int, +) -> bytes: + """Return one canonical gzip-compressed USTAR sdist.""" + + epoch = _validated_epoch(source_date_epoch) + members = _read_members(_decompress_sdist(payload)) + canonical_tar = _serialize_tar(members, source_date_epoch=epoch) + destination = io.BytesIO() + try: + with gzip.GzipFile( + filename="", + mode="wb", + compresslevel=9, + fileobj=destination, + mtime=epoch, + ) as compressed: + compressed.write(canonical_tar) + except (OSError, ValueError, zlib.error): + _fail("sdist_not_canonicalizable") + return destination.getvalue() + + +def _read_regular_file(path: Path, *, maximum: int) -> bytes: + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_nlink != 1 + or not 0 < before.st_size <= maximum + ): + _fail("artifact_invalid") + chunks: list[bytes] = [] + consumed = 0 + while consumed < before.st_size: + chunk = os.read(descriptor, min(1_048_576, before.st_size - consumed)) + if not chunk: + _fail("artifact_invalid") + chunks.append(chunk) + consumed += len(chunk) + after = os.fstat(descriptor) + if consumed != before.st_size or ( + after.st_dev, + after.st_ino, + after.st_mode, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) != ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ): + _fail("artifact_invalid") + return b"".join(chunks) + except ReleaseBuildError: + raise + except OSError: + _fail("artifact_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + +def _write_new_file(path: Path, payload: bytes, *, mode: int = 0o644) -> None: + descriptor = -1 + created = False + try: + descriptor = os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC | os.O_NOFOLLOW, + mode, + ) + created = True + offset = 0 + while offset < len(payload): + written = os.write(descriptor, payload[offset:]) + if written <= 0: + raise OSError + offset += written + os.fchmod(descriptor, mode) + os.fsync(descriptor) + if stat.S_IMODE(os.fstat(descriptor).st_mode) != mode or os.fstat( + descriptor + ).st_size != len(payload): + raise OSError + except OSError: + if created: + try: + path.unlink(missing_ok=True) + except OSError: + pass + _fail("artifact_write_failed") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + +def normalize_sdist( + source: Path, + destination: Path, + *, + source_date_epoch: int, +) -> None: + """Normalize one sdist path into a new destination file.""" + + if not source.name.endswith(".tar.gz") or destination.name != source.name: + _fail("sdist_path_invalid") + payload = _read_regular_file(source, maximum=_MAX_COMPRESSED_SDIST_BYTES) + normalized = normalize_sdist_bytes( + payload, + source_date_epoch=source_date_epoch, + ) + _write_new_file(destination, normalized) + + +def _sdist_member_inventory(payload: bytes) -> list[dict[str, object]]: + members = _read_members(_decompress_sdist(payload)) + inventory: list[dict[str, object]] = [] + for member in members: + if member.is_directory: + inventory.append( + { + "kind": "directory", + "mode": "0755", + "path": member.name, + "size_bytes": 0, + } + ) + else: + assert member.payload is not None + inventory.append( + { + "kind": "file", + "mode": "0755" if member.executable else "0644", + "path": member.name, + "sha256": hashlib.sha256(member.payload).hexdigest(), + "size_bytes": len(member.payload), + } + ) + return inventory + + +def artifact_inventory( + artifacts: Sequence[Path], + *, + source_date_epoch: int, + source_commit: str, +) -> bytes: + """Render canonical unsigned integrity evidence for release artifacts. + + The inventory binds artifact bytes to a builder-observed Git commit. It is + deliberately not represented as a signature or third-party attestation. + """ + + epoch = _validated_epoch(source_date_epoch) + commit = _validated_source_commit(source_commit) + records: dict[str, dict[str, object]] = {} + for path in artifacts: + if not isinstance(path, Path): + _fail("artifact_invalid") + name = path.name + if not name or name in records or PurePosixPath(name).name != name: + _fail("artifact_invalid") + if name.endswith(".whl"): + kind = "wheel" + elif name.endswith(".tar.gz"): + kind = "sdist" + else: + _fail("artifact_invalid") + payload = _read_regular_file(path, maximum=_MAX_ARTIFACT_BYTES) + record: dict[str, object] = { + "filename": name, + "kind": kind, + "sha256": hashlib.sha256(payload).hexdigest(), + "size_bytes": len(payload), + } + if kind == "sdist": + record["members"] = _sdist_member_inventory(payload) + records[name] = record + if len(records) != 2 or {record["kind"] for record in records.values()} != { + "wheel", + "sdist", + }: + _fail("artifact_invalid") + document = { + "artifacts": [records[name] for name in sorted(records)], + "schema": INVENTORY_SCHEMA, + "source_commit": commit, + "source_date_epoch": epoch, + } + return ( + json.dumps( + document, + ensure_ascii=True, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("ascii") + + b"\n" + ) + + +def sanitized_build_environment( + *, + source_date_epoch: int, + home: Path, +) -> Mapping[str, str]: + """Return the complete, minimal environment used for the backend process.""" + + epoch = _validated_epoch(source_date_epoch) + if not home.is_absolute(): + _fail("build_path_invalid") + return { + "HOME": str(home), + "LANG": "C", + "LC_ALL": "C", + "PATH": os.defpath, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INDEX": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", + "PYTHONNOUSERSITE": "1", + "SOURCE_DATE_EPOCH": str(epoch), + "TZ": "UTC", + } + + +def _git_binary() -> Path: + located = shutil.which("git", path=os.defpath) + if located is None: + _fail("git_unavailable") + try: + path = Path(located).resolve(strict=True) + status = path.stat() + except (OSError, RuntimeError): + _fail("git_unavailable") + if ( + not stat.S_ISREG(status.st_mode) + or not status.st_mode & stat.S_IXUSR + or status.st_mode & (stat.S_ISUID | stat.S_ISGID) + ): + _fail("git_unavailable") + return path + + +def _git_environment(home: Path) -> Mapping[str, str]: + if not home.is_absolute(): + _fail("build_path_invalid") + return { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0", + "HOME": str(home), + "LANG": "C", + "LC_ALL": "C", + "PATH": os.defpath, + } + + +def _run_git( + project: Path, + home: Path, + arguments: Sequence[str], +) -> bytes: + try: + completed = subprocess.run( + (str(_git_binary()), *arguments), + cwd=project, + env=dict(_git_environment(home)), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + timeout=_GIT_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired): + _fail("source_repository_invalid") + if completed.returncode != 0 or len(completed.stdout) > _MAX_GIT_OUTPUT_BYTES: + _fail("source_repository_invalid") + return completed.stdout + + +def _git_source_commit(project: Path, home: Path) -> str: + top_level_payload = _run_git( + project, + home, + ("rev-parse", "--show-toplevel"), + ) + try: + top_level_text = top_level_payload.decode(sys.getfilesystemencoding()) + top_level = Path(top_level_text.removesuffix("\n")).resolve(strict=True) + except (OSError, RuntimeError, UnicodeError): + _fail("source_repository_invalid") + if ( + not top_level_payload.endswith(b"\n") + or top_level_payload.endswith(b"\n\n") + or top_level != project + ): + _fail("source_repository_invalid") + + commit_payload = _run_git( + project, + home, + ("rev-parse", "--verify", "HEAD^{commit}"), + ) + try: + commit = commit_payload.decode("ascii").removesuffix("\n") + except UnicodeError: + _fail("source_repository_invalid") + if ( + not commit_payload.endswith(b"\n") + or commit_payload.endswith(b"\n\n") + or len(commit_payload) != 41 + ): + _fail("source_repository_invalid") + try: + validated_commit = _validated_source_commit(commit) + except ReleaseBuildError: + _fail("source_repository_invalid") + + status = _run_git( + project, + home, + ("status", "--porcelain=v1", "--untracked-files=all"), + ) + if status: + _fail("worktree_dirty") + return validated_commit + + +def _materialize_source_snapshot( + archive_payload: bytes, + destination: Path, +) -> None: + members = _read_members(archive_payload) + if not members: + _fail("source_archive_invalid") + try: + destination.mkdir(mode=0o700) + for member in members: + path = destination / member.name + if member.is_directory: + path.mkdir(mode=0o755, parents=True, exist_ok=False) + else: + path.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + assert member.payload is not None + _write_new_file( + path, + member.payload, + mode=0o755 if member.executable else 0o644, + ) + except ReleaseBuildError: + raise + except OSError: + _fail("source_archive_invalid") + + +def _export_clean_head( + project: Path, + destination: Path, + archive_path: Path, + home: Path, +) -> str: + commit = _git_source_commit(project, home) + if ( + destination.exists() + or archive_path.exists() + or destination.is_relative_to(project) + or archive_path.is_relative_to(project) + ): + _fail("build_path_invalid") + _run_git( + project, + home, + ( + "archive", + "--format=tar", + f"--output={archive_path}", + commit, + ), + ) + archive_payload = _read_regular_file( + archive_path, + maximum=_MAX_TAR_BYTES, + ) + _materialize_source_snapshot(archive_payload, destination) + try: + archive_path.unlink() + except OSError: + _fail("source_archive_invalid") + return commit + + +def _backend_artifacts(directory: Path) -> tuple[Path, Path]: + try: + entries = tuple(sorted(directory.iterdir(), key=lambda path: path.name)) + except OSError: + _fail("backend_output_invalid") + if len(entries) != 2 or any( + entry.is_symlink() or not entry.is_file() for entry in entries + ): + _fail("backend_output_invalid") + wheels = tuple(path for path in entries if path.name.endswith(".whl")) + sdists = tuple(path for path in entries if path.name.endswith(".tar.gz")) + if len(wheels) != 1 or len(sdists) != 1: + _fail("backend_output_invalid") + return wheels[0], sdists[0] + + +def _build_python_path(python_executable: Path | None) -> Path: + """Keep a virtual-environment launcher path without dereferencing it.""" + + candidate = Path(sys.executable) if python_executable is None else python_executable + try: + python = Path(os.path.abspath(candidate)) + except (OSError, TypeError, ValueError): + _fail("build_path_invalid") + if not python.is_absolute() or not python.is_file(): + _fail("build_path_invalid") + return python + + +def build_release( + project_root: Path, + output_directory: Path, + *, + source_date_epoch: int, + python_executable: Path | None = None, +) -> bytes: + """Run the backend once and atomically publish wheel, sdist, and inventory.""" + + epoch = _validated_epoch(source_date_epoch) + try: + project = project_root.resolve(strict=True) + output = output_directory.resolve(strict=False) + except (OSError, RuntimeError): + _fail("build_path_invalid") + if ( + not project.is_dir() + or not output.is_absolute() + or output.exists() + or output.is_symlink() + or output == project + or output.is_relative_to(project) + ): + _fail("build_path_invalid") + python = _build_python_path(python_executable) + try: + with tempfile.TemporaryDirectory( + prefix="netveil-build-", + ) as temporary: + workspace = Path(temporary) + backend = workspace / "backend" + home = workspace / "home" + source = workspace / "source" + archive_path = workspace / "source.tar" + for directory in (backend, home): + directory.mkdir(mode=0o700) + source_commit = _export_clean_head( + project, + source, + archive_path, + home, + ) + command = ( + str(python), + "-m", + "build", + "--no-isolation", + "--outdir", + str(backend), + str(source), + ) + try: + completed = subprocess.run( + command, + cwd=source, + env=dict( + sanitized_build_environment( + source_date_epoch=epoch, + home=home, + ) + ), + stdin=subprocess.DEVNULL, + capture_output=True, + check=False, + timeout=_BUILD_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired): + _fail("backend_failed") + if completed.returncode != 0: + _fail("backend_failed") + if _git_source_commit(project, home) != source_commit: + _fail("source_commit_changed") + wheel, sdist = _backend_artifacts(backend) + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists() or output.is_symlink(): + _fail("build_path_invalid") + with tempfile.TemporaryDirectory( + prefix=f".{output.name}.publish-", + dir=output.parent, + ) as publication_temporary: + release = Path(publication_temporary) / "release" + release.mkdir(mode=0o700) + wheel_payload = _read_regular_file( + wheel, + maximum=_MAX_ARTIFACT_BYTES, + ) + _write_new_file(release / wheel.name, wheel_payload) + normalize_sdist( + sdist, + release / sdist.name, + source_date_epoch=epoch, + ) + inventory = artifact_inventory( + (release / wheel.name, release / sdist.name), + source_date_epoch=epoch, + source_commit=source_commit, + ) + _write_new_file(release / INVENTORY_FILENAME, inventory) + os.rename(release, output) + return inventory + except ReleaseBuildError: + raise + except OSError: + _fail("artifact_write_failed") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Build a wheel and canonical sdist without dependency isolation, " + "then emit deterministic SHA-256 inventory evidence." + ) + ) + parser.add_argument("project_root", type=Path) + parser.add_argument("output_directory", type=Path) + parser.add_argument( + "--source-date-epoch", + type=int, + required=True, + ) + parser.add_argument("--python", type=Path) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + namespace = _parser().parse_args(argv) + try: + inventory = build_release( + namespace.project_root, + namespace.output_directory, + source_date_epoch=namespace.source_date_epoch, + python_executable=namespace.python, + ) + except ReleaseBuildError as error: + print(str(error), file=sys.stderr) + return 2 + sys.stdout.buffer.write(inventory) + sys.stdout.buffer.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/render_evidence.py b/tools/render_evidence.py new file mode 100755 index 0000000..7e12651 --- /dev/null +++ b/tools/render_evidence.py @@ -0,0 +1,1220 @@ +#!/usr/bin/env python3 +"""Render Netveil's committed release evidence as deterministic inert visuals.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import os +import stat +import sys +import tempfile +import textwrap +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, NoReturn, cast + +ROOT: Final = Path(__file__).resolve().parents[1] +VERIFICATION_PATH: Final = ROOT / "docs/evidence/fresh-wheel-verification.json" +INVENTORY_PATH: Final = ROOT / "docs/evidence/release-inventory.json" +GENERATOR_PATH: Final = ROOT / "tools/render_evidence.py" + +CLI_SVG_PATH: Final = "docs/assets/cli-session.svg" +COUNTS_SVG_PATH: Final = "docs/assets/receipt-counts.svg" +MATRIX_SVG_PATH: Final = "docs/assets/verification-matrix.svg" +PROVENANCE_SVG_PATH: Final = "docs/assets/artifact-provenance.svg" +CAST_PATH: Final = "docs/evidence/cli-session.cast" +MANIFEST_PATH: Final = "docs/evidence/visual-manifest.json" + +VISUAL_OUTPUT_PATHS: Final = ( + CLI_SVG_PATH, + COUNTS_SVG_PATH, + MATRIX_SVG_PATH, + PROVENANCE_SVG_PATH, + CAST_PATH, +) +ALL_OUTPUT_PATHS: Final = (*VISUAL_OUTPUT_PATHS, MANIFEST_PATH) + +VERIFICATION_SCHEMA: Final = "netveil.fresh-wheel-verification.v1" +INVENTORY_SCHEMA: Final = "netveil.release-inventory.v1" +VISUAL_MANIFEST_SCHEMA: Final = "netveil.visual-evidence.v1" +MAX_INPUT_BYTES: Final = 16 * 1_048_576 +LOWER_HEX: Final = frozenset("0123456789abcdef") +RAW_ENDPOINTS: Final = ( + "192.0.2.10", + "198.51.100.20", + "203.0.113.30", + "2001:db8::10", +) + +BACKGROUND: Final = "#07111f" +PANEL: Final = "#101d31" +PANEL_ALT: Final = "#0c1728" +BORDER: Final = "#29405e" +TEXT: Final = "#f4f8ff" +MUTED: Final = "#a8b7c9" +CYAN: Final = "#67e8f9" +VIOLET: Final = "#c4b5fd" +GREEN: Final = "#86efac" +AMBER: Final = "#fcd34d" +PINK: Final = "#fda4af" +MONO: Final = "DejaVu Sans Mono,ui-monospace,SFMono-Regular,Consolas,monospace" +SANS: Final = "DejaVu Sans,Arial,sans-serif" + + +class EvidenceRenderError(RuntimeError): + """The evidence bundle is absent, malformed, inconsistent, or stale.""" + + +@dataclass(frozen=True, slots=True) +class ArtifactFact: + """One release artifact represented in both evidence documents.""" + + filename: str + kind: str + sha256: str + size_bytes: int + member_count: int + + +@dataclass(frozen=True, slots=True) +class TraceFact: + """Path-free normalized syscall evidence.""" + + label: str + sha256: str + exec_count: int + exit_count: int + network_count: int + child_count: int + + +@dataclass(frozen=True, slots=True) +class EvidenceFacts: + """Validated facts used by every generated output.""" + + source_commit: str + source_date_epoch: int + inventory_sha256: str + artifacts: tuple[ArtifactFact, ...] + checks: tuple[str, ...] + interpreter: str + platform: str + launcher_mode: str + launcher_sha256: str + launcher_size_bytes: int + record_mode: str + record_sha256: str + record_size_bytes: int + selected_record_rows: int + traces: tuple[TraceFact, ...] + version_stdout: str + receipt: dict[str, object] + receipt_stdout: bytes + endpoint_occurrences: int + unique_endpoints: int + physical_lines: int + source_bytes: int + ipv4_occurrences: int + ipv6_occurrences: int + documentation_occurrences: int + duplicate_groups: int + + +def _fail(message: str) -> NoReturn: + raise EvidenceRenderError(message) + + +def _canonical_json(document: object) -> bytes: + return json.dumps( + document, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def _json_bytes(document: object) -> bytes: + return _canonical_json(document) + b"\n" + + +def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + output: dict[str, object] = {} + for key, value in pairs: + if key in output: + _fail("JSON contains a duplicate object key") + output[key] = value + return output + + +def _load_canonical_json(payload: bytes, label: str) -> dict[str, object]: + if ( + type(payload) is not bytes + or not 0 < len(payload) <= MAX_INPUT_BYTES + or not payload.endswith(b"\n") + or payload.endswith(b"\n\n") + ): + _fail(f"{label} is not one bounded canonical JSON line") + try: + document = json.loads( + payload.decode("ascii"), + object_pairs_hook=_unique_object, + parse_constant=lambda value: _fail( + f"{label} contains invalid constant {value}" + ), + ) + except EvidenceRenderError: + raise + except (UnicodeError, json.JSONDecodeError, ValueError) as error: + raise EvidenceRenderError(f"{label} is not valid ASCII JSON") from error + if not isinstance(document, dict) or _json_bytes(document) != payload: + _fail(f"{label} is not canonical") + return cast(dict[str, object], document) + + +def _mapping(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict): + _fail(f"{label} must be an object") + return cast(dict[str, object], value) + + +def _mapping_at( + document: Mapping[str, object], + key: str, + label: str, +) -> dict[str, object]: + if key not in document: + _fail(f"{label} is missing {key}") + return _mapping(document[key], f"{label}.{key}") + + +def _sequence(value: object, label: str) -> list[object]: + if not isinstance(value, list): + _fail(f"{label} must be an array") + return cast(list[object], value) + + +def _sequence_at( + document: Mapping[str, object], + key: str, + label: str, +) -> list[object]: + if key not in document: + _fail(f"{label} is missing {key}") + return _sequence(document[key], f"{label}.{key}") + + +def _text(value: object, label: str, *, allow_newline: bool = False) -> str: + if not isinstance(value, str) or not value.isascii(): + _fail(f"{label} must be ASCII text") + forbidden = "\x00\r" if allow_newline else "\x00\r\n" + if any(character in forbidden for character in value): + _fail(f"{label} contains a forbidden control character") + if any( + ord(character) < 32 and (allow_newline is False or character != "\n") + for character in value + ): + _fail(f"{label} contains a control character") + return value + + +def _text_at( + document: Mapping[str, object], + key: str, + label: str, + *, + allow_newline: bool = False, +) -> str: + if key not in document: + _fail(f"{label} is missing {key}") + return _text( + document[key], + f"{label}.{key}", + allow_newline=allow_newline, + ) + + +def _integer(value: object, label: str) -> int: + if type(value) is not int or value < 0: + _fail(f"{label} must be a non-negative integer") + return value + + +def _integer_at( + document: Mapping[str, object], + key: str, + label: str, +) -> int: + if key not in document: + _fail(f"{label} is missing {key}") + return _integer(document[key], f"{label}.{key}") + + +def _false_at(document: Mapping[str, object], key: str, label: str) -> None: + if document.get(key) is not False: + _fail(f"{label}.{key} must be false") + + +def _hex(value: object, label: str, length: int) -> str: + text = _text(value, label) + if len(text) != length or any(character not in LOWER_HEX for character in text): + _fail(f"{label} is not lowercase hexadecimal") + return text + + +def _mode(value: object, label: str) -> str: + text = _text(value, label) + if len(text) != 4 or any(character not in "01234567" for character in text): + _fail(f"{label} is not a four-digit mode") + return text + + +def _parse_artifacts( + artifacts: list[object], +) -> tuple[ArtifactFact, ...]: + parsed: list[ArtifactFact] = [] + for index, item in enumerate(artifacts): + artifact = _mapping(item, f"artifacts[{index}]") + filename = _text_at(artifact, "filename", "artifact") + kind = _text_at(artifact, "kind", "artifact") + if kind not in ("wheel", "sdist"): + _fail("artifact kind must be wheel or sdist") + members = ( + _sequence_at(artifact, "members", "artifact") if kind == "sdist" else [] + ) + if kind == "wheel" and "members" in artifact: + _fail("release inventory wheel must not claim sdist members") + parsed.append( + ArtifactFact( + filename=filename, + kind=kind, + sha256=_hex(artifact.get("sha256"), "artifact.sha256", 64), + size_bytes=_integer_at(artifact, "size_bytes", "artifact"), + member_count=len(members), + ) + ) + if ( + len(parsed) != 2 + or {artifact.kind for artifact in parsed} != {"wheel", "sdist"} + or [artifact.filename for artifact in parsed] + != sorted(artifact.filename for artifact in parsed) + ): + _fail("release artifacts are not the exact sorted wheel/sdist pair") + return tuple(parsed) + + +def _parse_checks(raw_checks: list[object]) -> tuple[str, ...]: + checks: list[str] = [] + for index, item in enumerate(raw_checks): + check = _mapping(item, f"checks[{index}]") + if set(check) != {"name", "status"} or check.get("status") != "pass": + _fail("every evidence check must have exact pass status") + name = _text_at(check, "name", "check") + if not name or any( + character not in "abcdefghijklmnopqrstuvwxyz_" for character in name + ): + _fail("check names must be lowercase identifiers") + checks.append(name) + required = { + "coordinated_bootstrap_record_mutation_accepted", + "public_demo_capture", + "release_inventory_integrity", + "source_commit_bound", + "syscall_trace_offline", + "tamper_fail_closed", + } + if ( + len(checks) < len(required) + or len(checks) != len(set(checks)) + or not required.issubset(checks) + ): + _fail("verification check inventory is incomplete") + return tuple(checks) + + +def _parse_traces(raw_traces: list[object]) -> tuple[TraceFact, ...]: + traces: list[TraceFact] = [] + for index, item in enumerate(raw_traces): + trace = _mapping(item, f"syscall_traces[{index}]") + chain = [ + _text(value, "trace.exec_chain item") + for value in _sequence_at(trace, "exec_chain", "trace") + ] + if chain != ["installed_launcher", "installed_python"]: + _fail("trace exec chain is not the exact launcher/Python pair") + traces.append( + TraceFact( + label=_text_at(trace, "label", "trace"), + sha256=_hex( + trace.get("normalized_sha256"), + "trace.normalized_sha256", + 64, + ), + exec_count=_integer_at(trace, "exec_count", "trace"), + exit_count=_integer_at(trace, "exit_syscall_count", "trace"), + network_count=_integer_at( + trace, + "network_syscall_count", + "trace", + ), + child_count=_integer_at( + trace, + "post_launch_process_count", + "trace", + ), + ) + ) + if {trace.label for trace in traces} != {"receipt", "version"} or any( + trace.exec_count != 2 + or trace.exit_count < 1 + or trace.network_count != 0 + or trace.child_count != 0 + for trace in traces + ): + _fail("normalized trace evidence is incomplete") + return tuple(sorted(traces, key=lambda trace: trace.label)) + + +def _parse_demo( + demo: Mapping[str, object], +) -> tuple[ + str, + dict[str, object], + bytes, + int, + int, + int, + int, + int, + int, + int, + int, +]: + if ( + demo.get("classification") + != "synthetic_ietf_documentation_ranges_with_public_demo_key" + ): + _fail("public demo classification is not exact") + corpus = _mapping_at(demo, "corpus", "public_demo") + _hex(corpus.get("sha256"), "public_demo.corpus.sha256", 64) + _integer_at(corpus, "size_bytes", "public_demo.corpus") + physical_lines = _integer_at(corpus, "physical_lines", "public_demo.corpus") + key = _mapping_at(demo, "public_demo_key", "public_demo") + if ( + key.get("classification") != "public_non_secret_test_material" + or key.get("source_constant") != "tools/verify_fresh_wheel.py:_PUBLIC_DEMO_KEY" + or _integer_at(key, "size_bytes", "public_demo.public_demo_key") != 32 + ): + _fail("public demo key classification is incomplete") + _hex(key.get("sha256"), "public_demo.public_demo_key.sha256", 64) + + commands = _sequence_at(demo, "commands", "public_demo") + if len(commands) != 2: + _fail("public demo must contain exactly two commands") + version = _mapping(commands[0], "public_demo.commands[0]") + receipt_command = _mapping(commands[1], "public_demo.commands[1]") + expected_version_argv = ["netveil-audit", "--version"] + expected_receipt_argv = [ + "netveil-audit", + "receipt", + "documentation-corpus.txt", + "--key-file", + "public-demo.key", + ] + for command, expected_argv in ( + (version, expected_version_argv), + (receipt_command, expected_receipt_argv), + ): + argv = [ + _text(value, "public demo argv item") + for value in _sequence_at(command, "argv", "public demo command") + ] + if ( + argv != expected_argv + or _integer_at(command, "exit_code", "public demo command") != 0 + or _text_at( + command, + "stderr", + "public demo command", + allow_newline=True, + ) + != "" + ): + _fail("public demo command contract changed") + version_stdout = _text_at( + version, + "stdout", + "public demo version", + allow_newline=True, + ) + if version_stdout != "netveil-audit 0.3.0\n": + _fail("public demo version output changed") + + receipt = _mapping_at(receipt_command, "stdout_json", "public demo receipt") + receipt_stdout = _json_bytes(receipt) + if hashlib.sha256(receipt_stdout).hexdigest() != _hex( + receipt_command.get("stdout_sha256"), + "public demo receipt stdout_sha256", + 64, + ): + _fail("public demo receipt stdout digest does not match") + if receipt.get("schema") != "netveil.aggregate-receipt.v1": + _fail("public demo receipt schema changed") + report = _mapping_at(receipt, "report", "public demo receipt") + counts = _mapping_at(report, "counts", "public demo report") + by_version = _mapping_at( + report, + "endpoint_occurrences_by_ip_version", + "public demo report", + ) + by_scope = _mapping_at( + report, + "endpoint_occurrences_by_scope", + "public demo report", + ) + duplicates = _mapping_at(report, "duplicates", "public demo report") + return ( + version_stdout, + receipt, + receipt_stdout, + _integer_at(counts, "endpoint_occurrences", "receipt counts"), + _integer_at(counts, "unique_endpoints", "receipt counts"), + physical_lines, + _integer_at(counts, "source_bytes", "receipt counts"), + _integer_at(by_version, "ipv4", "receipt by version"), + _integer_at(by_version, "ipv6", "receipt by version"), + _integer_at(by_scope, "documentation", "receipt by scope"), + _integer_at(duplicates, "group_count", "receipt duplicates"), + ) + + +def parse_evidence( + verification_payload: bytes, + inventory_payload: bytes, +) -> EvidenceFacts: + """Validate the cross-document evidence contract and return renderable facts.""" + + verification = _load_canonical_json( + verification_payload, + "fresh-wheel verification", + ) + inventory = _load_canonical_json(inventory_payload, "release inventory") + if ( + verification.get("schema") != VERIFICATION_SCHEMA + or verification.get("status") != "pass" + or inventory.get("schema") != INVENTORY_SCHEMA + ): + _fail("evidence schema or status is not publishable") + source_commit = _hex( + verification.get("source_commit"), + "verification.source_commit", + 40, + ) + if inventory.get("source_commit") != source_commit: + _fail("verification and inventory source commits differ") + source_date_epoch = _integer_at( + inventory, + "source_date_epoch", + "release inventory", + ) + inventory_artifacts = _sequence_at( + inventory, + "artifacts", + "release inventory", + ) + artifacts = _parse_artifacts(inventory_artifacts) + + integrity = _mapping_at( + verification, + "integrity_evidence", + "verification", + ) + if ( + integrity.get("artifacts") != inventory_artifacts + or integrity.get("inventory_schema") != INVENTORY_SCHEMA + or integrity.get("inventory_type") != "unsigned_sha256_manifest" + or integrity.get("source_commit") != source_commit + or integrity.get("source_date_epoch") != source_date_epoch + ): + _fail("verification does not embed the exact release inventory facts") + _false_at(integrity, "signature_verified", "integrity evidence") + _false_at(integrity, "attestation_verified", "integrity evidence") + inventory_sha256 = _hex( + integrity.get("inventory_sha256"), + "integrity_evidence.inventory_sha256", + 64, + ) + if inventory_sha256 != hashlib.sha256(inventory_payload).hexdigest(): + _fail("release inventory digest does not match its bytes") + + wheel = _mapping_at(verification, "wheel", "verification") + wheel_artifact = next( + artifact for artifact in artifacts if artifact.kind == "wheel" + ) + if ( + wheel.get("sha256") != wheel_artifact.sha256 + or wheel.get("size_bytes") != wheel_artifact.size_bytes + or not _sequence_at(wheel, "members", "verification wheel") + ): + _fail("wheel verification facts do not match the release inventory") + + installed = _mapping_at(verification, "installed", "verification") + launcher = _mapping_at(installed, "launcher", "installed") + record = _mapping_at(installed, "record", "installed") + rows = _sequence_at(installed, "selected_record_rows", "installed") + if not rows: + _fail("installed RECORD evidence is empty") + + interpreter = _mapping_at(verification, "interpreter", "verification") + platform = _mapping_at(verification, "platform", "verification") + demo_values = _parse_demo(_mapping_at(verification, "public_demo", "verification")) + ( + version_stdout, + receipt, + receipt_stdout, + endpoint_occurrences, + unique_endpoints, + physical_lines, + source_bytes, + ipv4_occurrences, + ipv6_occurrences, + documentation_occurrences, + duplicate_groups, + ) = demo_values + + if any( + endpoint.encode("ascii") in verification_payload for endpoint in RAW_ENDPOINTS + ): + _fail("public evidence unexpectedly contains raw endpoint text") + + return EvidenceFacts( + source_commit=source_commit, + source_date_epoch=source_date_epoch, + inventory_sha256=inventory_sha256, + artifacts=artifacts, + checks=_parse_checks(_sequence_at(verification, "checks", "verification")), + interpreter=( + f"{_text_at(interpreter, 'implementation', 'interpreter')} " + f"{_text_at(interpreter, 'version', 'interpreter')} · " + f"{_text_at(interpreter, 'cache_tag', 'interpreter')}" + ), + platform=( + f"{_text_at(platform, 'system', 'platform')} " + f"{_text_at(platform, 'release', 'platform')} · " + f"{_text_at(platform, 'machine', 'platform')}" + ), + launcher_mode=_mode(launcher.get("mode"), "installed launcher mode"), + launcher_sha256=_hex( + launcher.get("sha256"), + "installed launcher sha256", + 64, + ), + launcher_size_bytes=_integer_at( + launcher, + "size_bytes", + "installed launcher", + ), + record_mode=_mode(record.get("mode"), "installed RECORD mode"), + record_sha256=_hex( + record.get("sha256"), + "installed RECORD sha256", + 64, + ), + record_size_bytes=_integer_at(record, "size_bytes", "installed RECORD"), + selected_record_rows=len(rows), + traces=_parse_traces( + _sequence_at(verification, "syscall_traces", "verification") + ), + version_stdout=version_stdout, + receipt=receipt, + receipt_stdout=receipt_stdout, + endpoint_occurrences=endpoint_occurrences, + unique_endpoints=unique_endpoints, + physical_lines=physical_lines, + source_bytes=source_bytes, + ipv4_occurrences=ipv4_occurrences, + ipv6_occurrences=ipv6_occurrences, + documentation_occurrences=documentation_occurrences, + duplicate_groups=duplicate_groups, + ) + + +def _human_bytes(value: int) -> str: + if value < 1_024: + return f"{value} B" + if value < 1_048_576: + return f"{value / 1_024:.1f} KiB" + return f"{value / 1_048_576:.1f} MiB" + + +def _svg_document( + *, + title: str, + description: str, + width: int, + height: int, + body: Sequence[str], +) -> bytes: + escaped_title = html.escape(title) + escaped_description = html.escape(description) + lines = [ + ( + f'' + ), + f'{escaped_title}', + f'{escaped_description}', + "", + f'', + *body, + "", + ] + return ("\n".join(lines) + "\n").encode("utf-8") + + +def _terminal_transcript(facts: EvidenceFacts) -> str: + receipt = facts.receipt_stdout.decode("ascii").rstrip("\n") + return ( + "Netveil installed-wheel public demonstration\n" + f"source commit: {facts.source_commit}\n" + "input class: synthetic IETF documentation ranges\n" + "key class: public non-secret test material\n\n" + "$ netveil-audit --version\n" + f"{facts.version_stdout}" + "$ netveil-audit receipt documentation-corpus.txt " + "--key-file public-demo.key\n" + f"{receipt}\n" + ) + + +def _render_cli(facts: EvidenceFacts) -> bytes: + wrapped: list[str] = [] + for line in _terminal_transcript(facts).rstrip("\n").splitlines(): + wrapped.extend( + textwrap.wrap( + line, + width=126, + subsequent_indent=" ", + break_long_words=True, + break_on_hyphens=False, + replace_whitespace=False, + drop_whitespace=False, + ) + or [""] + ) + line_height = 21 + height = 245 + len(wrapped) * line_height + body = [ + 'Installed CLI · captured stdout', + ( + 'Exact successful output ' + "captured by the fresh-wheel verifier; long canonical JSON is " + "visually wrapped only." + ), + ( + f'' + ), + '', + '', + '', + ] + y = 180 + for line in wrapped: + fill = CYAN if line.startswith("$ ") else TEXT + body.append( + f'' + f"{html.escape(line)}" + ) + y += line_height + return _svg_document( + title="Netveil installed CLI evidence", + description=( + "Terminal rendering of exact version and receipt standard output " + "captured from the verified fresh wheel." + ), + width=1440, + height=height, + body=body, + ) + + +def _render_counts(facts: EvidenceFacts) -> bytes: + metrics = ( + ("physical lines", facts.physical_lines, CYAN), + ("endpoint occurrences", facts.endpoint_occurrences, VIOLET), + ("unique endpoints", facts.unique_endpoints, GREEN), + ("IPv4 occurrences", facts.ipv4_occurrences, "#38bdf8"), + ("IPv6 occurrences", facts.ipv6_occurrences, "#f472b6"), + ("duplicate groups", facts.duplicate_groups, AMBER), + ) + maximum = max(value for _, value, _ in metrics) + body = [ + 'Public receipt · disclosed counts', + ( + 'Values parsed from the exact ' + "verifier-captured receipt; identifiers and raw endpoints are not " + "used in this chart." + ), + '', + ] + y = 175 + for label, value, color in metrics: + width = 820 * value / maximum if maximum else 0 + body.extend( + ( + f'{html.escape(label)}', + f'', + f'', + f'{value}', + ) + ) + y += 62 + body.extend( + ( + '', + 'Scope result', + ( + f'{facts.documentation_occurrences} / ' + f"{facts.endpoint_occurrences} occurrences · documentation ranges" + ), + ( + 'Synthetic corpus only; this ' + "is workflow evidence, not a claim about live infrastructure." + ), + '', + 'Disclosure boundary', + ( + f'{facts.source_bytes} source bytes ' + "are summarized, not embedded" + ), + ( + 'Counts, equality and ' + "frequency remain public; pseudonymized does not mean anonymous." + ), + ) + ) + return _svg_document( + title="Netveil public receipt count evidence", + description=( + "Bar chart of physical lines, endpoint counts, IP versions, and " + "duplicate groups parsed from the committed public demo receipt." + ), + width=1440, + height=830, + body=body, + ) + + +def _render_matrix(facts: EvidenceFacts) -> bytes: + columns = 3 + rows = (len(facts.checks) + columns - 1) // columns + height = 500 + rows * 58 + body = [ + 'Fresh-wheel verification matrix', + ( + f'{len(facts.checks)} exact ' + "checks reported pass by the committed verifier JSON." + ), + '', + (f'{html.escape(facts.interpreter)}'), + f'{html.escape(facts.platform)}', + ( + f'' + f"{facts.source_commit}" + ), + ( + '' + "unsigned consistency evidence · not attestation" + ), + ] + for index, check in enumerate(facts.checks): + column = index % columns + row = index // columns + x = 48 + column * 448 + y = 270 + row * 58 + body.extend( + ( + f'', + f'', + ( + f'' + f"{html.escape(check.replace('_', ' '))}" + ), + f'PASS', + ) + ) + trace_y = 300 + rows * 58 + body.append( + f'Normalized offline syscall evidence' + ) + for index, trace in enumerate(facts.traces): + x = 48 + index * 680 + body.extend( + ( + f'', + ( + f'' + f"{html.escape(trace.label)}" + ), + ( + f'' + f"exec={trace.exec_count} · network={trace.network_count} · " + f"post-launch children={trace.child_count}" + ), + ( + f'' + f"normalized sha256 {trace.sha256}" + ), + ) + ) + return _svg_document( + title="Netveil fresh-wheel verification matrix", + description=( + "All verifier pass labels, interpreter and platform identity, and " + "normalized path-free process and network trace evidence." + ), + width=1440, + height=height, + body=body, + ) + + +def _render_provenance(facts: EvidenceFacts) -> bytes: + wheel = next(artifact for artifact in facts.artifacts if artifact.kind == "wheel") + sdist = next(artifact for artifact in facts.artifacts if artifact.kind == "sdist") + boxes = ( + ( + 48, + "1 · clean Git HEAD", + facts.source_commit, + f"SOURCE_DATE_EPOCH {facts.source_date_epoch}", + CYAN, + ), + ( + 390, + "2 · deterministic artifacts", + f"wheel · {_human_bytes(wheel.size_bytes)} · {wheel.sha256[:20]}…", + ( + f"sdist · {_human_bytes(sdist.size_bytes)} · " + f"{sdist.member_count} members" + ), + VIOLET, + ), + ( + 732, + "3 · fresh installed state", + ( + f"launcher {facts.launcher_mode} · " + f"{_human_bytes(facts.launcher_size_bytes)}" + ), + ( + f"RECORD {facts.record_mode} · " + f"{facts.selected_record_rows} selected rows" + ), + GREEN, + ), + ( + 1074, + "4 · executed evidence", + f"{len(facts.checks)} checks pass", + "2 traces · network 0 · children 0", + AMBER, + ), + ) + body = [ + 'Release evidence chain', + ( + 'Every displayed value comes ' + "from the canonical release inventory or fresh-wheel verifier JSON." + ), + ] + for index, (x, title, line_one, line_two, color) in enumerate(boxes): + body.extend( + ( + f'', + f'', + f'{html.escape(title)}', + f'{html.escape(line_one)}', + f'{html.escape(line_two)}', + ( + f'BOUND IN EVIDENCE' + if index + else f'BUILDER-OBSERVED' + ), + ) + ) + if index < len(boxes) - 1: + body.extend( + ( + f'', + f'', + ) + ) + body.extend( + ( + '', + 'Exact installed bindings', + ( + f'launcher sha256 ' + f"{facts.launcher_sha256}" + ), + ( + f'RECORD sha256 ' + f"{facts.record_sha256}" + ), + ( + f'inventory sha256 ' + f"{facts.inventory_sha256}" + ), + '', + 'Claim boundary', + ( + 'The manifest is unsigned. ' + "It shows internally consistent bytes and an executed test run; " + "it does not prove publisher identity, an honest host, or an " + "unmodified verifier." + ), + ( + 'Authenticate the published ' + "artifact digests through a separate trusted channel." + ), + ) + ) + return _svg_document( + title="Netveil release evidence chain", + description=( + "Source commit, deterministic artifact, installed-file, and " + "fresh execution evidence with an explicit unsigned non-claim." + ), + width=1440, + height=825, + body=body, + ) + + +def _render_cast(facts: EvidenceFacts) -> bytes: + command_one = "$ netveil-audit --version\r\n" + command_two = ( + "$ netveil-audit receipt documentation-corpus.txt " + "--key-file public-demo.key\r\n" + ) + receipt = facts.receipt_stdout.decode("ascii").replace("\n", "\r\n") + header = { + "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}, + "height": 40, + "timestamp": facts.source_date_epoch, + "version": 2, + "width": 120, + } + events: tuple[object, ...] = ( + [ + 0.1, + "o", + ( + "Netveil verified public demo\r\n" + "synthetic IETF documentation ranges · public non-secret key\r\n\r\n" + ), + ], + [0.6, "o", command_one], + [0.9, "o", facts.version_stdout.replace("\n", "\r\n")], + [1.4, "o", command_two], + [1.8, "o", receipt], + ) + lines = [_canonical_json(header), *(_canonical_json(event) for event in events)] + return b"\n".join(lines) + b"\n" + + +def render_bundle( + verification_payload: bytes, + inventory_payload: bytes, + *, + generator_payload: bytes, +) -> dict[str, bytes]: + """Return every generated output, including the non-self-referential manifest.""" + + facts = parse_evidence(verification_payload, inventory_payload) + outputs = { + CLI_SVG_PATH: _render_cli(facts), + COUNTS_SVG_PATH: _render_counts(facts), + MATRIX_SVG_PATH: _render_matrix(facts), + PROVENANCE_SVG_PATH: _render_provenance(facts), + CAST_PATH: _render_cast(facts), + } + manifest = { + "claim_boundary": ( + "unsigned internally consistent build-and-execution evidence; " + "not publisher authentication, host attestation, or a signature" + ), + "generator": { + "path": "tools/render_evidence.py", + "sha256": hashlib.sha256(generator_payload).hexdigest(), + "size_bytes": len(generator_payload), + }, + "inputs": [ + { + "path": "docs/evidence/fresh-wheel-verification.json", + "sha256": hashlib.sha256(verification_payload).hexdigest(), + "size_bytes": len(verification_payload), + }, + { + "path": "docs/evidence/release-inventory.json", + "sha256": hashlib.sha256(inventory_payload).hexdigest(), + "size_bytes": len(inventory_payload), + }, + ], + "outputs": [ + { + "path": path, + "sha256": hashlib.sha256(outputs[path]).hexdigest(), + "size_bytes": len(outputs[path]), + } + for path in sorted(outputs) + ], + "schema": VISUAL_MANIFEST_SCHEMA, + "source_commit": facts.source_commit, + } + outputs[MANIFEST_PATH] = _json_bytes(manifest) + return outputs + + +def _read_input(path: Path, label: str) -> bytes: + try: + status = path.lstat() + if ( + not stat.S_ISREG(status.st_mode) + or not 0 < status.st_size <= MAX_INPUT_BYTES + ): + _fail(f"{label} is not one bounded regular file") + payload = path.read_bytes() + except OSError as error: + raise EvidenceRenderError(f"{label} cannot be read") from error + if len(payload) != status.st_size: + _fail(f"{label} changed while being read") + return payload + + +def _assert_safe_target(path: Path) -> None: + try: + relative = path.relative_to(ROOT) + except ValueError: + _fail("generated output escaped the repository") + current = ROOT + for part in relative.parts[:-1]: + current /= part + if current.exists() and current.is_symlink(): + _fail(f"generated output parent is a symlink: {relative.as_posix()}") + if path.exists(): + try: + mode = path.lstat().st_mode + except OSError as error: + raise EvidenceRenderError("generated output cannot be inspected") from error + if not stat.S_ISREG(mode): + _fail(f"generated output is not regular: {relative.as_posix()}") + + +def _replace_file(path: Path, payload: bytes) -> None: + _assert_safe_target(path) + try: + path.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + descriptor, raw_temporary = tempfile.mkstemp( + prefix=f".{path.name}.", + dir=path.parent, + ) + temporary = Path(raw_temporary) + try: + os.fchmod(descriptor, 0o644) + offset = 0 + while offset < len(payload): + written = os.write(descriptor, payload[offset:]) + if written <= 0: + raise OSError("write made no progress") + offset += written + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, path) + except OSError as error: + try: + temporary.unlink(missing_ok=True) + except (OSError, UnboundLocalError): + pass + raise EvidenceRenderError(f"cannot publish {path.name}") from error + + +def _check_outputs(outputs: Mapping[str, bytes]) -> None: + stale: list[str] = [] + for relative_path in ALL_OUTPUT_PATHS: + path = ROOT / relative_path + _assert_safe_target(path) + try: + observed = path.read_bytes() + except OSError: + stale.append(relative_path) + continue + if observed != outputs[relative_path]: + stale.append(relative_path) + if stale: + _fail("stale generated evidence: " + ", ".join(stale)) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Render deterministic SVG and terminal-cast views from committed " + "Netveil release evidence." + ) + ) + parser.add_argument( + "--check", + action="store_true", + help="fail unless every committed generated byte is current", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + namespace = _parser().parse_args(argv) + try: + verification_payload = _read_input( + VERIFICATION_PATH, + "fresh-wheel verification", + ) + inventory_payload = _read_input(INVENTORY_PATH, "release inventory") + generator_payload = _read_input(GENERATOR_PATH, "evidence renderer") + outputs = render_bundle( + verification_payload, + inventory_payload, + generator_payload=generator_payload, + ) + if namespace.check: + _check_outputs(outputs) + else: + for relative_path in VISUAL_OUTPUT_PATHS: + _replace_file(ROOT / relative_path, outputs[relative_path]) + _replace_file(ROOT / MANIFEST_PATH, outputs[MANIFEST_PATH]) + except EvidenceRenderError as error: + print(f"netveil-evidence-renderer: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify_fresh_wheel.py b/tools/verify_fresh_wheel.py new file mode 100755 index 0000000..9d53e56 --- /dev/null +++ b/tools/verify_fresh_wheel.py @@ -0,0 +1,2472 @@ +#!/usr/bin/env python3 +"""Verify one pinned Netveil wheel in a disposable, offline virtual environment. + +The verifier never builds a distribution and never resolves dependencies. It +first pins and inspects the exact wheel bytes supplied by the caller, then asks +pip to install a private copy with ``--no-index --no-deps``. Product processes +run with bounded output capture and receive only synthetic IETF documentation +range inputs. +""" + +from __future__ import annotations + +import base64 +import csv +import gzip +import hashlib +import importlib.metadata +import importlib.util +import io +import json +import marshal +import os +import re +import shutil +import signal +import stat +import struct +import subprocess +import sys +import tarfile +import tempfile +import venv +import zlib +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from email import policy +from email.parser import BytesParser +from pathlib import Path, PurePosixPath +from types import CodeType +from typing import Final, NoReturn +from zipfile import BadZipFile, ZipFile, ZipInfo + +_DISTRIBUTION_NAME: Final = "netveil-audit" +_DISTRIBUTION_VERSION: Final = "0.3.0" +_WHEEL_NAME: Final = "netveil_audit-0.3.0-py3-none-any.whl" +_SDIST_NAME: Final = "netveil_audit-0.3.0.tar.gz" +_DIST_INFO: Final = "netveil_audit-0.3.0.dist-info" +_WHEEL_SCRIPT: Final = "netveil_audit-0.3.0.data/scripts/netveil-audit" +_LAUNCHER_NAME: Final = "netveil-audit" +_BOOTSTRAP_NAME: Final = "netveil_bootstrap.py" +_SCHEMA: Final = "netveil.fresh-wheel-verification.v1" +_INVENTORY_SCHEMA: Final = "netveil.release-inventory.v1" +_INVENTORY_NAME: Final = "release-inventory.json" + +_MAX_WHEEL_BYTES: Final = 32 * 1_048_576 +_MAX_INVENTORY_BYTES: Final = 16 * 1_048_576 +_MAX_RELEASE_ARTIFACT_BYTES: Final = 512 * 1_048_576 +_MAX_COMPRESSED_SDIST_BYTES: Final = 128 * 1_048_576 +_MAX_EXPANDED_SDIST_BYTES: Final = 256 * 1_048_576 +_MAX_SDIST_MEMBER_BYTES: Final = 64 * 1_048_576 +_MAX_SDIST_FILE_BYTES: Final = 192 * 1_048_576 +_MAX_SDIST_MEMBERS: Final = 20_000 +_MAX_MEMBER_BYTES: Final = 2 * 1_048_576 +_MAX_EXPANDED_BYTES: Final = 16 * 1_048_576 +_MAX_PROCESS_OUTPUT_BYTES: Final = 1_048_576 +_PROCESS_TIMEOUT_SECONDS: Final = 30.0 +_ARTIFACT_FAILURE: Final = b"netveil-audit: artifact_unverified\n" +_LOWER_HEX: Final = frozenset("0123456789abcdef") +_TAR_BLOCK_BYTES: Final = 512 +_TAR_TRAILER_BYTES: Final = 2 * _TAR_BLOCK_BYTES + +_EXPECTED_WHEEL_MEMBERS: Final = frozenset( + { + _WHEEL_SCRIPT, + "netveil/__init__.py", + "netveil/cli.py", + "netveil/model.py", + "netveil/parser.py", + "netveil/privacy.py", + "netveil/py.typed", + _BOOTSTRAP_NAME, + f"{_DIST_INFO}/METADATA", + f"{_DIST_INFO}/RECORD", + f"{_DIST_INFO}/WHEEL", + f"{_DIST_INFO}/licenses/LICENSE", + f"{_DIST_INFO}/top_level.txt", + } +) +_CHECKS: Final = ( + "wheel_archive", + "fresh_install", + "entry_points_absent", + "launcher_identity", + "direct_and_path_commands", + "isolated_startup", + "environment_injection_inert", + "unchecked_bytecode_inert", + "unknown_bytecode_rejected", + "tamper_fail_closed", + "metadata_drift_rejected", + "coordinated_bootstrap_record_mutation_accepted", + "unknown_package_file_rejected", + "receipt_deterministic", + "receipt_redacted", + "public_demo_capture", + "syscall_trace_offline", + "release_inventory_integrity", + "source_commit_bound", +) +_CORPUS: Final = ( + b"# Synthetic IETF documentation ranges only\n" + b"192.0.2.10:443\n" + b"192.0.2.10:443\n" + b"198.51.100.20:80\n" + b"203.0.113.30:65535\n" + b"[2001:db8::10]:8443\n" +) +# Deliberately public, deterministic demonstration material. It is never +# suitable for a private corpus; its sole purpose is reproducible CLI output +# over the synthetic IETF documentation-range corpus above. +_PUBLIC_DEMO_KEY: Final = b"netveil-public-demo-key-v1-00001" +_RAW_ENDPOINT_TOKENS: Final = ( + b"192.0.2.10", + b"198.51.100.20", + b"203.0.113.30", + b"2001:db8::10", +) +_ATTACK_SOURCE: Final = ( + b"import os\n" + b"from pathlib import Path\n" + b'Path(os.environ["NETVEIL_VERIFY_MARKER"]).write_bytes(b"executed")\n' +) +_PROBE_SOURCE: Final = b"""\ +import json +import sys +from pathlib import Path + +def main(argv): + root = str(Path(__file__).parent) + document = { + "argv_exact": list(argv) == ["isolation-probe"], + "dont_write_bytecode": sys.flags.dont_write_bytecode, + "empty_path_absent": "" not in sys.path, + "ignore_environment": sys.flags.ignore_environment, + "implementation": sys.implementation.name, + "isolated": sys.flags.isolated, + "netveil_package_absent": not any( + name == "netveil" or name.startswith("netveil.") + for name in sys.modules + ), + "no_site": sys.flags.no_site, + "safe_path": sys.flags.safe_path, + "cache_tag": sys.implementation.cache_tag, + "site_root_count": sys.path.count(root), + "version": "{}.{}.{}".format(*sys.version_info[:3]), + } + sys.stdout.write( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\\n" + ) + sys.stdout.flush() + return 0 +""" +_EXPECTED_PROBE: Final = { + "argv_exact": True, + "dont_write_bytecode": 1, + "empty_path_absent": True, + "ignore_environment": 1, + "netveil_package_absent": True, + "isolated": 1, + "no_site": 1, + "safe_path": True, + "site_root_count": 1, +} + + +class VerificationFailure(RuntimeError): + """A stable, deliberately redacted verification failure.""" + + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +@dataclass(frozen=True, slots=True) +class RecordEntry: + """One parsed wheel or installed RECORD row.""" + + path: str + digest: str | None + size: int | None + + +@dataclass(frozen=True, slots=True) +class WheelMemberEvidence: + """One archive member bound into the public verification evidence.""" + + path: str + sha256: str + size: int + mode: str + + +@dataclass(frozen=True, slots=True) +class WheelEvidence: + """Pinned evidence extracted without installing the wheel.""" + + payload: bytes + sha256: str + launcher: bytes + members: tuple[WheelMemberEvidence, ...] + + +@dataclass(frozen=True, slots=True) +class InventoryArtifactEvidence: + """One artifact digest asserted by the unsigned builder inventory.""" + + filename: str + kind: str + sha256: str + size_bytes: int + members: tuple[InventoryMemberEvidence, ...] | None + + +@dataclass(frozen=True, slots=True) +class InventoryMemberEvidence: + """One safe logical member of the canonical release sdist.""" + + path: str + kind: str + mode: str + size_bytes: int + sha256: str | None + + +@dataclass(frozen=True, slots=True) +class ReleaseInventoryEvidence: + """Strictly parsed unsigned integrity inventory from the release builder.""" + + source_commit: str + source_date_epoch: int + sha256: str + artifacts: tuple[InventoryArtifactEvidence, ...] + + +@dataclass(frozen=True, slots=True) +class ProcessResult: + """Bounded child-process result.""" + + returncode: int + stdout: bytes + stderr: bytes + + +@dataclass(frozen=True, slots=True) +class InstalledLayout: + """Paths inside one disposable installation.""" + + prefix: Path + python: Path + launcher: Path + site_root: Path + dist_info: Path + bootstrap: Path + package_root: Path + record: Path + evidence: InstalledEvidence | None = None + + +@dataclass(frozen=True, slots=True) +class InstalledRecordRowEvidence: + """One selected installed RECORD binding with no private filesystem path.""" + + path: str + sha256: str | None + size_bytes: int | None + + +@dataclass(frozen=True, slots=True) +class InstalledFileEvidence: + """Stable content and mode evidence for one installed regular file.""" + + logical_path: str + mode: str + sha256: str + size_bytes: int + + +@dataclass(frozen=True, slots=True) +class InstalledEvidence: + """Path-free evidence captured from the fresh installed artifact.""" + + launcher: InstalledFileEvidence + record: InstalledFileEvidence + selected_record_rows: tuple[InstalledRecordRowEvidence, ...] + + +@dataclass(frozen=True, slots=True) +class InterpreterEvidence: + """Path-free identity reported by the isolated installed interpreter.""" + + implementation: str + version: str + cache_tag: str + + +@dataclass(frozen=True, slots=True) +class PlatformEvidence: + """Hostname-free verifier platform identity.""" + + sys_platform: str + system: str + release: str + machine: str + + +@dataclass(frozen=True, slots=True) +class TraceEvidence: + """Normalized path-free evidence from one process/network syscall trace.""" + + label: str + normalized_sha256: str + process_count: int + exec_chain: tuple[str, ...] + exec_count: int + exit_syscall_count: int + network_syscall_count: int + post_launch_process_count: int + + +@dataclass(frozen=True, slots=True) +class ReceiptEvidence: + """Private paths retained only for the syscall trace gate.""" + + corpus: Path + key: Path + output: bytes + + +@dataclass(frozen=True, slots=True) +class PublicDemoEvidence: + """Reproducible public CLI output over synthetic, non-secret inputs.""" + + corpus_sha256: str + corpus_size_bytes: int + corpus_physical_lines: int + public_key_sha256: str + public_key_size_bytes: int + version_stdout: str + receipt: dict[str, object] + receipt_stdout_sha256: str + + +@dataclass(frozen=True, slots=True) +class VerificationSummary: + """Stable evidence safe to render outside the private workspace.""" + + source_commit: str + release_inventory: ReleaseInventoryEvidence + installed: InstalledEvidence + interpreter: InterpreterEvidence + platform: PlatformEvidence + syscall_traces: tuple[TraceEvidence, ...] + public_demo: PublicDemoEvidence + wheel_sha256: str + wheel_size_bytes: int + wheel_members: tuple[WheelMemberEvidence, ...] + + def document(self) -> dict[str, object]: + artifact_documents: list[dict[str, object]] = [] + for artifact in self.release_inventory.artifacts: + artifact_document: dict[str, object] = { + "filename": artifact.filename, + "kind": artifact.kind, + "sha256": artifact.sha256, + "size_bytes": artifact.size_bytes, + } + if artifact.members is not None: + member_documents: list[dict[str, object]] = [] + for member in artifact.members: + member_document: dict[str, object] = { + "kind": member.kind, + "mode": member.mode, + "path": member.path, + "size_bytes": member.size_bytes, + } + if member.sha256 is not None: + member_document["sha256"] = member.sha256 + member_documents.append(member_document) + artifact_document["members"] = member_documents + artifact_documents.append(artifact_document) + + def installed_file_document( + evidence: InstalledFileEvidence, + ) -> dict[str, object]: + return { + "logical_path": evidence.logical_path, + "mode": evidence.mode, + "sha256": evidence.sha256, + "size_bytes": evidence.size_bytes, + } + + return { + "checks": [{"name": name, "status": "pass"} for name in _CHECKS], + "installed": { + "launcher": installed_file_document(self.installed.launcher), + "record": installed_file_document(self.installed.record), + "selected_record_rows": [ + { + "path": row.path, + "sha256": row.sha256, + "size_bytes": row.size_bytes, + } + for row in self.installed.selected_record_rows + ], + }, + "integrity_evidence": { + "artifacts": artifact_documents, + "attestation_verified": False, + "inventory_schema": _INVENTORY_SCHEMA, + "inventory_sha256": self.release_inventory.sha256, + "inventory_type": "unsigned_sha256_manifest", + "signature_verified": False, + "source_commit": self.release_inventory.source_commit, + "source_date_epoch": self.release_inventory.source_date_epoch, + }, + "interpreter": { + "cache_tag": self.interpreter.cache_tag, + "implementation": self.interpreter.implementation, + "version": self.interpreter.version, + }, + "platform": { + "machine": self.platform.machine, + "release": self.platform.release, + "sys_platform": self.platform.sys_platform, + "system": self.platform.system, + }, + "public_demo": { + "classification": ( + "synthetic_ietf_documentation_ranges_with_public_demo_key" + ), + "commands": [ + { + "argv": ["netveil-audit", "--version"], + "exit_code": 0, + "stderr": "", + "stdout": self.public_demo.version_stdout, + }, + { + "argv": [ + "netveil-audit", + "receipt", + "documentation-corpus.txt", + "--key-file", + "public-demo.key", + ], + "exit_code": 0, + "stderr": "", + "stdout_json": self.public_demo.receipt, + "stdout_sha256": self.public_demo.receipt_stdout_sha256, + }, + ], + "corpus": { + "physical_lines": self.public_demo.corpus_physical_lines, + "sha256": self.public_demo.corpus_sha256, + "size_bytes": self.public_demo.corpus_size_bytes, + }, + "public_demo_key": { + "classification": "public_non_secret_test_material", + "sha256": self.public_demo.public_key_sha256, + "size_bytes": self.public_demo.public_key_size_bytes, + "source_constant": ("tools/verify_fresh_wheel.py:_PUBLIC_DEMO_KEY"), + }, + }, + "schema": _SCHEMA, + "source_commit": self.source_commit, + "status": "pass", + "syscall_traces": [ + { + "exec_chain": list(trace.exec_chain), + "exec_count": trace.exec_count, + "exit_syscall_count": trace.exit_syscall_count, + "label": trace.label, + "network_syscall_count": trace.network_syscall_count, + "normalized_sha256": trace.normalized_sha256, + "post_launch_process_count": trace.post_launch_process_count, + "process_count": trace.process_count, + } + for trace in self.syscall_traces + ], + "wheel": { + "members": [ + { + "mode": member.mode, + "path": member.path, + "sha256": member.sha256, + "size_bytes": member.size, + } + for member in self.wheel_members + ], + "sha256": self.wheel_sha256, + "size_bytes": self.wheel_size_bytes, + }, + } + + +def _fail(code: str) -> NoReturn: + raise VerificationFailure(code) + + +def _identity(value: os.stat_result) -> tuple[int, ...]: + return ( + value.st_dev, + value.st_ino, + value.st_mode, + value.st_uid, + value.st_gid, + value.st_nlink, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _read_exact_inventory(path: Path) -> bytes: + if os.name != "posix" or path.name != _INVENTORY_NAME: + _fail("inventory_path_invalid") + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if any(not hasattr(os, name) for name in required): + _fail("platform_unsupported") + flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + descriptor = -1 + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size <= 0 + or before.st_size > _MAX_INVENTORY_BYTES + ): + _fail("inventory_path_invalid") + chunks: list[bytes] = [] + observed = 0 + while observed <= before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail("inventory_path_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("inventory_path_invalid") + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail("inventory_path_changed") + return payload + + +def _reject_json_constant(_: str) -> NoReturn: + _fail("inventory_invalid") + + +def _unique_json_object( + pairs: list[tuple[str, object]], +) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + _fail("inventory_invalid") + document[key] = value + return document + + +def _parse_inventory_members( + raw_members: object, +) -> tuple[InventoryMemberEvidence, ...]: + if ( + not isinstance(raw_members, list) + or not 1 <= len(raw_members) <= _MAX_SDIST_MEMBERS + ): + _fail("inventory_invalid") + members: list[InventoryMemberEvidence] = [] + total_file_bytes = 0 + for raw_member in raw_members: + if not isinstance(raw_member, dict): + _fail("inventory_invalid") + kind = raw_member.get("kind") + expected_keys = ( + {"kind", "mode", "path", "size_bytes", "sha256"} + if kind == "file" + else {"kind", "mode", "path", "size_bytes"} + ) + if set(raw_member) != expected_keys: + _fail("inventory_invalid") + path_text = raw_member["path"] + mode = raw_member["mode"] + size_bytes = raw_member["size_bytes"] + sha256 = raw_member.get("sha256") + if not isinstance(path_text, str): + _fail("inventory_invalid") + path = PurePosixPath(path_text) + first = path.parts[0] if path.parts else "" + if ( + not path_text + or "\\" in path_text + or path.is_absolute() + or path.as_posix() != path_text + or any(part in ("", ".", "..") for part in path.parts) + or (len(first) >= 2 and first[0].isalpha() and first[1] == ":") + or any( + ord(character) < 32 or ord(character) == 127 for character in path_text + ) + or kind not in ("directory", "file") + or not isinstance(mode, str) + or type(size_bytes) is not int + ): + _fail("inventory_invalid") + if kind == "directory": + if mode != "0755" or size_bytes != 0 or sha256 is not None: + _fail("inventory_invalid") + else: + if ( + mode not in ("0644", "0755") + or not 0 <= size_bytes <= _MAX_SDIST_MEMBER_BYTES + or not isinstance(sha256, str) + or len(sha256) != 64 + or any(character not in _LOWER_HEX for character in sha256) + ): + _fail("inventory_invalid") + total_file_bytes += size_bytes + if total_file_bytes > _MAX_SDIST_FILE_BYTES: + _fail("inventory_invalid") + members.append( + InventoryMemberEvidence( + path=path_text, + kind=kind, + mode=mode, + size_bytes=size_bytes, + sha256=sha256, + ) + ) + paths = [member.path for member in members] + if paths != sorted(paths) or len(paths) != len(set(paths)): + _fail("inventory_invalid") + return tuple(members) + + +def _parse_release_inventory(payload: bytes) -> ReleaseInventoryEvidence: + if ( + type(payload) is not bytes + or not payload.endswith(b"\n") + or payload.endswith(b"\n\n") + ): + _fail("inventory_not_canonical") + try: + text = payload.decode("ascii") + document = json.loads( + text, + object_pairs_hook=_unique_json_object, + parse_constant=_reject_json_constant, + ) + except VerificationFailure: + raise + except (UnicodeError, json.JSONDecodeError, ValueError): + _fail("inventory_invalid") + if _canonical_json(document) + b"\n" != payload: + _fail("inventory_not_canonical") + if ( + not isinstance(document, dict) + or set(document) + != { + "artifacts", + "schema", + "source_commit", + "source_date_epoch", + } + or document["schema"] != _INVENTORY_SCHEMA + ): + _fail("inventory_invalid") + source_commit = document["source_commit"] + if not isinstance(source_commit, str): + _fail("inventory_invalid") + try: + _validate_source_commit(source_commit) + except VerificationFailure: + _fail("inventory_invalid") + source_date_epoch = document["source_date_epoch"] + if ( + type(source_date_epoch) is not int + or not 0 <= source_date_epoch <= (1 << 32) - 1 + ): + _fail("inventory_invalid") + raw_artifacts = document["artifacts"] + if not isinstance(raw_artifacts, list) or len(raw_artifacts) != 2: + _fail("inventory_invalid") + artifacts: list[InventoryArtifactEvidence] = [] + for raw_artifact in raw_artifacts: + if not isinstance(raw_artifact, dict): + _fail("inventory_invalid") + kind = raw_artifact.get("kind") + expected_keys = {"filename", "kind", "sha256", "size_bytes"} + if kind == "sdist": + expected_keys.add("members") + if set(raw_artifact) != expected_keys: + _fail("inventory_invalid") + filename = raw_artifact["filename"] + sha256 = raw_artifact["sha256"] + size_bytes = raw_artifact["size_bytes"] + if ( + not isinstance(filename, str) + or not filename + or "\\" in filename + or PurePosixPath(filename).name != filename + or any( + ord(character) < 32 or ord(character) == 127 for character in filename + ) + or kind not in ("wheel", "sdist") + or not isinstance(sha256, str) + or len(sha256) != 64 + or any(character not in _LOWER_HEX for character in sha256) + or type(size_bytes) is not int + or not 0 < size_bytes <= _MAX_RELEASE_ARTIFACT_BYTES + ): + _fail("inventory_invalid") + members = ( + _parse_inventory_members(raw_artifact["members"]) + if kind == "sdist" + else None + ) + artifacts.append( + InventoryArtifactEvidence( + filename=filename, + kind=kind, + sha256=sha256, + size_bytes=size_bytes, + members=members, + ) + ) + if [artifact.filename for artifact in artifacts] != sorted( + artifact.filename for artifact in artifacts + ) or {artifact.kind for artifact in artifacts} != {"wheel", "sdist"}: + _fail("inventory_invalid") + return ReleaseInventoryEvidence( + source_commit=source_commit, + source_date_epoch=source_date_epoch, + sha256=hashlib.sha256(payload).hexdigest(), + artifacts=tuple(artifacts), + ) + + +def _bind_release_inventory( + inventory: ReleaseInventoryEvidence, + *, + expected_source_commit: str, + wheel_path: Path, + wheel_payload: bytes, + sdist_path: Path, + sdist_payload: bytes, +) -> None: + if inventory.source_commit != expected_source_commit: + _fail("inventory_source_commit_mismatch") + by_kind = {artifact.kind: artifact for artifact in inventory.artifacts} + wheel = by_kind["wheel"] + sdist = by_kind["sdist"] + if wheel.filename != _WHEEL_NAME or wheel_path.name != wheel.filename: + _fail("inventory_wheel_name_mismatch") + if sdist.filename != _SDIST_NAME: + _fail("inventory_sdist_name_mismatch") + if sdist_path.name != sdist.filename: + _fail("inventory_sdist_name_mismatch") + if wheel.size_bytes != len(wheel_payload): + _fail("inventory_wheel_size_mismatch") + if wheel.sha256 != hashlib.sha256(wheel_payload).hexdigest(): + _fail("inventory_wheel_sha256_mismatch") + if sdist.size_bytes != len(sdist_payload): + _fail("inventory_sdist_size_mismatch") + if sdist.sha256 != hashlib.sha256(sdist_payload).hexdigest(): + _fail("inventory_sdist_sha256_mismatch") + + +def _bind_sdist_members( + inventory: ReleaseInventoryEvidence, + actual_members: tuple[InventoryMemberEvidence, ...], +) -> None: + sdist = next( + artifact for artifact in inventory.artifacts if artifact.kind == "sdist" + ) + if sdist.members != actual_members: + _fail("inventory_sdist_members_mismatch") + + +def _read_exact_wheel(path: Path) -> bytes: + if os.name != "posix" or path.name != _WHEEL_NAME: + _fail("wheel_path_invalid") + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if any(not hasattr(os, name) for name in required): + _fail("platform_unsupported") + flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + descriptor = -1 + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size <= 0 + or before.st_size > _MAX_WHEEL_BYTES + ): + _fail("wheel_path_invalid") + chunks: list[bytes] = [] + observed = 0 + while observed <= before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail("wheel_path_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("wheel_path_invalid") + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail("wheel_path_changed") + return payload + + +def _read_exact_sdist(path: Path) -> bytes: + if os.name != "posix" or path.name != _SDIST_NAME: + _fail("sdist_path_invalid") + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if any(not hasattr(os, name) for name in required): + _fail("platform_unsupported") + flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + descriptor = -1 + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size <= 0 + or before.st_size > _MAX_COMPRESSED_SDIST_BYTES + ): + _fail("sdist_path_invalid") + chunks: list[bytes] = [] + observed = 0 + while observed <= before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail("sdist_path_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("sdist_path_invalid") + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail("sdist_path_changed") + return payload + + +def _safe_sdist_member_name(member: tarfile.TarInfo) -> str: + raw_name = member.name + if ( + not isinstance(raw_name, str) + or not raw_name + or "\\" in raw_name + or "\x00" in raw_name + or any(ord(character) < 32 or ord(character) == 127 for character in raw_name) + ): + _fail("sdist_member_unsafe") + normalized = ( + raw_name[:-1] if member.isdir() and raw_name.endswith("/") else raw_name + ) + path = PurePosixPath(normalized) + first = path.parts[0] if path.parts else "" + if ( + not normalized + or path.is_absolute() + or path.as_posix() != normalized + or any(part in ("", ".", "..") for part in path.parts) + or (len(first) >= 2 and first[0].isalpha() and first[1] == ":") + or (not member.isdir() and raw_name.endswith("/")) + ): + _fail("sdist_member_unsafe") + return normalized + + +def _inspect_sdist( + payload: bytes, + *, + source_date_epoch: int, +) -> tuple[InventoryMemberEvidence, ...]: + if ( + type(payload) is not bytes + or len(payload) < 18 + or payload[:4] != b"\x1f\x8b\x08\x00" + or int.from_bytes(payload[4:8], "little") != source_date_epoch + ): + _fail("sdist_not_canonical") + compressed = io.BytesIO(payload) + try: + with gzip.GzipFile(fileobj=compressed, mode="rb") as archive: + tar_payload = archive.read(_MAX_EXPANDED_SDIST_BYTES + 1) + except (EOFError, OSError, gzip.BadGzipFile, zlib.error): + _fail("sdist_archive_invalid") + if ( + not 0 < len(tar_payload) <= _MAX_EXPANDED_SDIST_BYTES + or len(tar_payload) % _TAR_BLOCK_BYTES + or len(tar_payload) < _TAR_TRAILER_BYTES + ): + _fail("sdist_archive_invalid") + + observed: list[InventoryMemberEvidence] = [] + observed_paths: set[str] = set() + total_file_bytes = 0 + archive_offset = -1 + try: + with tarfile.open(fileobj=io.BytesIO(tar_payload), mode="r:") as archive: + members = archive.getmembers() + archive_offset = archive.offset + if not 1 <= len(members) <= _MAX_SDIST_MEMBERS: + _fail("sdist_archive_invalid") + for member in members: + if ( + not (member.isdir() or member.isreg()) + or member.sparse is not None + or member.uid != 0 + or member.gid != 0 + or member.uname != "" + or member.gname != "" + or member.linkname + or member.pax_headers + or member.mtime != source_date_epoch + ): + _fail("sdist_member_unsafe") + name = _safe_sdist_member_name(member) + if name in observed_paths: + _fail("sdist_member_duplicate") + observed_paths.add(name) + if member.isdir(): + if member.size != 0 or member.mode != 0o755: + _fail("sdist_not_canonical") + observed.append( + InventoryMemberEvidence( + path=name, + kind="directory", + mode="0755", + size_bytes=0, + sha256=None, + ) + ) + continue + if ( + not 0 <= member.size <= _MAX_SDIST_MEMBER_BYTES + or member.mode not in (0o644, 0o755) + ): + _fail("sdist_member_unsafe") + total_file_bytes += member.size + if total_file_bytes > _MAX_SDIST_FILE_BYTES: + _fail("sdist_archive_invalid") + extracted = archive.extractfile(member) + if extracted is None: + _fail("sdist_archive_invalid") + member_payload = extracted.read(_MAX_SDIST_MEMBER_BYTES + 1) + if len(member_payload) != member.size: + _fail("sdist_archive_invalid") + observed.append( + InventoryMemberEvidence( + path=name, + kind="file", + mode=f"{member.mode:04o}", + size_bytes=len(member_payload), + sha256=hashlib.sha256(member_payload).hexdigest(), + ) + ) + except VerificationFailure: + raise + except (OSError, tarfile.TarError, UnicodeError, ValueError): + _fail("sdist_archive_invalid") + if ( + archive_offset < 0 + or len(tar_payload) - archive_offset < _TAR_TRAILER_BYTES + or any(tar_payload[archive_offset:]) + or [member.path for member in observed] + != sorted(member.path for member in observed) + ): + _fail("sdist_not_canonical") + return tuple(observed) + + +def _validate_member_name(name: str) -> None: + path = PurePosixPath(name) + if ( + not name + or "\\" in name + or path.is_absolute() + or any(part in ("", ".", "..") for part in path.parts) + ): + _fail("wheel_archive_invalid") + + +def _validate_zip_member(info: ZipInfo) -> None: + _validate_member_name(info.filename) + mode = info.external_attr >> 16 + if ( + info.is_dir() + or info.flag_bits & 0x1 + or info.file_size < 0 + or info.file_size > _MAX_MEMBER_BYTES + or not mode + or not stat.S_ISREG(mode) + ): + _fail("wheel_archive_invalid") + + +def _decode_record_digest(value: str) -> str: + if not value.startswith("sha256="): + _fail("record_invalid") + digest = value.removeprefix("sha256=") + try: + decoded = base64.urlsafe_b64decode(digest + "=" * (-len(digest) % 4)) + except (UnicodeError, ValueError): + _fail("record_invalid") + if len(decoded) != hashlib.sha256().digest_size: + _fail("record_invalid") + return digest + + +def _parse_record(payload: bytes) -> dict[str, RecordEntry]: + try: + text = payload.decode("utf-8") + rows = list(csv.reader(io.StringIO(text, newline=""))) + except (UnicodeError, csv.Error): + _fail("record_invalid") + records: dict[str, RecordEntry] = {} + for row in rows: + if len(row) != 3: + _fail("record_invalid") + path, raw_digest, raw_size = row + if not path or path in records: + _fail("record_invalid") + digest = _decode_record_digest(raw_digest) if raw_digest else None + if raw_size: + if not raw_size.isascii() or not raw_size.isdecimal(): + _fail("record_invalid") + size: int | None = int(raw_size) + else: + size = None + if (digest is None) != (size is None): + _fail("record_invalid") + records[path] = RecordEntry(path=path, digest=digest, size=size) + return records + + +def _record_digest(payload: bytes) -> str: + digest = hashlib.sha256(payload).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def _assert_record_match(entry: RecordEntry, payload: bytes) -> None: + if entry.size != len(payload) or entry.digest != _record_digest(payload): + _fail("record_mismatch") + + +def _parse_metadata(payload: bytes) -> Mapping[str, str]: + message = BytesParser(policy=policy.default).parsebytes(payload) + if ( + message["Name"] != _DISTRIBUTION_NAME + or message["Version"] != _DISTRIBUTION_VERSION + or message["Requires-Python"] != ">=3.11" + or message.get_all("Requires-Dist", []) != [] + ): + _fail("wheel_metadata_invalid") + return dict(message.items()) + + +def _inspect_wheel(payload: bytes) -> WheelEvidence: + try: + with ZipFile(io.BytesIO(payload)) as archive: + infos = archive.infolist() + names = [info.filename for info in infos] + if len(names) != len(set(names)): + _fail("wheel_archive_invalid") + for info in infos: + _validate_zip_member(info) + if ( + frozenset(names) != _EXPECTED_WHEEL_MEMBERS + or sum(info.file_size for info in infos) > _MAX_EXPANDED_BYTES + ): + _fail("wheel_inventory_invalid") + members = {name: archive.read(name) for name in names} + member_info = {info.filename: info for info in infos} + except VerificationFailure: + raise + except (BadZipFile, KeyError, OSError, RuntimeError): + _fail("wheel_archive_invalid") + + _parse_metadata(members[f"{_DIST_INFO}/METADATA"]) + if members[f"{_DIST_INFO}/top_level.txt"] != b"netveil\nnetveil_bootstrap\n": + _fail("wheel_metadata_invalid") + wheel_metadata = BytesParser(policy=policy.default).parsebytes( + members[f"{_DIST_INFO}/WHEEL"] + ) + if ( + wheel_metadata["Wheel-Version"] != "1.0" + or wheel_metadata["Root-Is-Purelib"] != "true" + or wheel_metadata.get_all("Tag", []) != ["py3-none-any"] + ): + _fail("wheel_metadata_invalid") + + record_name = f"{_DIST_INFO}/RECORD" + records = _parse_record(members[record_name]) + if set(records) != set(members): + _fail("record_invalid") + for name, member_payload in members.items(): + entry = records[name] + if name == record_name: + if entry.digest is not None or entry.size is not None: + _fail("record_invalid") + else: + _assert_record_match(entry, member_payload) + + launcher = members[_WHEEL_SCRIPT] + launcher_mode = member_info[_WHEEL_SCRIPT].external_attr >> 16 + if not launcher_mode & stat.S_IXUSR: + _fail("launcher_invalid") + _validate_launcher_bytes(launcher) + return WheelEvidence( + payload=payload, + sha256=hashlib.sha256(payload).hexdigest(), + launcher=launcher, + members=tuple( + WheelMemberEvidence( + path=name, + sha256=hashlib.sha256(members[name]).hexdigest(), + size=len(members[name]), + mode=f"{stat.S_IMODE(member_info[name].external_attr >> 16):04o}", + ) + for name in sorted(members) + ), + ) + + +def _validate_launcher_bytes(payload: bytes) -> None: + if ( + not payload.startswith(b"#!/bin/sh\n") + or b'exec "$netveil_script_directory/python" -IESB "$0" "$@"\n' not in payload + or b"[project.scripts]" in payload + ): + _fail("launcher_invalid") + + +def _write_exclusive(path: Path, payload: bytes, *, mode: int) -> None: + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + mode, + ) + offset = 0 + while offset < len(payload): + written = os.write(descriptor, payload[offset:]) + if written <= 0: + _fail("private_file_write_failed") + offset += written + os.fsync(descriptor) + except OSError: + _fail("private_file_write_failed") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("private_file_write_failed") + + +def _base_environment( + root: Path, layout: InstalledLayout | None = None +) -> dict[str, str]: + home = root / "home" + temporary = root / "tmp" + home.mkdir(exist_ok=True) + temporary.mkdir(exist_ok=True) + binary = layout.launcher.parent if layout is not None else root + return { + "HOME": str(home), + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": str(binary), + "PIP_CONFIG_FILE": os.devnull, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INDEX": "1", + "PIP_NO_INPUT": "1", + "PIP_ROOT_USER_ACTION": "ignore", + "TMPDIR": str(temporary), + } + + +def _kill_process_group(process: subprocess.Popen[bytes]) -> None: + try: + os.killpg(process.pid, signal.SIGKILL) + except (OSError, ProcessLookupError): + try: + process.kill() + except OSError: + pass + + +def _run_process( + argv: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str], +) -> ProcessResult: + try: + with ( + tempfile.TemporaryFile("w+b") as stdout_file, + tempfile.TemporaryFile("w+b") as stderr_file, + ): + process = subprocess.Popen( + list(argv), + cwd=cwd, + env=dict(env), + stdin=subprocess.DEVNULL, + stdout=stdout_file, + stderr=stderr_file, + close_fds=True, + start_new_session=True, + ) + try: + returncode = process.wait(timeout=_PROCESS_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + _kill_process_group(process) + process.wait() + _fail("process_timeout") + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read(_MAX_PROCESS_OUTPUT_BYTES + 1) + stderr = stderr_file.read(_MAX_PROCESS_OUTPUT_BYTES + 1) + except OSError: + _fail("process_start_failed") + if ( + len(stdout) > _MAX_PROCESS_OUTPUT_BYTES + or len(stderr) > _MAX_PROCESS_OUTPUT_BYTES + ): + _fail("process_output_unbounded") + return ProcessResult(returncode=returncode, stdout=stdout, stderr=stderr) + + +def _create_fresh_install( + root: Path, + wheel: WheelEvidence, +) -> InstalledLayout: + pinned_wheel = root / _WHEEL_NAME + _write_exclusive(pinned_wheel, wheel.payload, mode=0o600) + prefix = root / "venv" + try: + venv.EnvBuilder( + system_site_packages=False, + clear=False, + symlinks=False, + with_pip=False, + ).create(prefix) + except (OSError, subprocess.SubprocessError): + _fail("venv_creation_failed") + python = prefix / "bin" / "python" + if not python.is_file(): + _fail("venv_creation_failed") + environment = _base_environment(root) + ensurepip = _run_process( + ( + str(python), + "-I", + "-m", + "ensurepip", + "--default-pip", + ), + cwd=root, + env=environment, + ) + if ensurepip.returncode != 0: + _fail("ensurepip_failed") + install = _run_process( + ( + str(python), + "-I", + "-m", + "pip", + "install", + "--no-index", + "--no-deps", + "--disable-pip-version-check", + "--no-cache-dir", + str(pinned_wheel), + ), + cwd=root, + env=environment, + ) + if install.returncode != 0: + _fail("wheel_install_failed") + return _inspect_install(prefix, python, wheel.launcher) + + +def _installed_record_path(site_root: Path, launcher: Path) -> str: + return PurePosixPath(os.path.relpath(launcher, site_root)).as_posix() + + +def _read_installed_regular( + path: Path, + *, + maximum: int, +) -> tuple[bytes, os.stat_result]: + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size <= 0 + or before.st_size > maximum + ): + _fail("installed_layout_invalid") + chunks: list[bytes] = [] + observed = 0 + while observed < before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size - observed), + ) + if not chunk: + _fail("installed_layout_invalid") + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except VerificationFailure: + raise + except OSError: + _fail("installed_layout_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("installed_layout_invalid") + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail("installed_layout_changed") + return payload, before + + +def _record_sha256_hex(digest: str | None) -> str | None: + if digest is None: + return None + try: + decoded = base64.urlsafe_b64decode(digest + "=" * (-len(digest) % 4)) + except (UnicodeError, ValueError): + _fail("record_invalid") + if len(decoded) != hashlib.sha256().digest_size: + _fail("record_invalid") + return decoded.hex() + + +def _inspect_install( + prefix: Path, + python: Path, + expected_launcher: bytes, +) -> InstalledLayout: + candidates = { + path.resolve() + for library in ("lib", "lib64") + for path in (prefix / library).glob(f"python*/site-packages/{_DIST_INFO}") + if path.is_dir() + } + if len(candidates) != 1: + _fail("installed_layout_invalid") + dist_info = candidates.pop() + site_root = dist_info.parent + launcher = prefix / "bin" / _LAUNCHER_NAME + bootstrap = site_root / _BOOTSTRAP_NAME + package_root = site_root / "netveil" + record_path = dist_info / "RECORD" + required = (launcher, bootstrap, package_root / "cli.py", record_path) + if any(not path.is_file() for path in required): + _fail("installed_layout_invalid") + launcher_payload, launcher_status = _read_installed_regular( + launcher, + maximum=_MAX_MEMBER_BYTES, + ) + record_payload, record_status = _read_installed_regular( + record_path, + maximum=_MAX_MEMBER_BYTES, + ) + if not launcher_status.st_mode & stat.S_IXUSR: + _fail("launcher_invalid") + if launcher_payload != expected_launcher: + _fail("launcher_identity_mismatch") + _validate_launcher_bytes(launcher_payload) + records = _parse_record(record_payload) + relative_launcher = _installed_record_path(site_root, launcher) + launcher_record = records.get(relative_launcher) + if launcher_record is None: + _fail("launcher_record_missing") + _assert_record_match(launcher_record, launcher_payload) + record_record_path = f"{_DIST_INFO}/RECORD" + record_record = records.get(record_record_path) + if ( + record_record is None + or record_record.digest is not None + or record_record.size is not None + ): + _fail("record_invalid") + if (dist_info / "entry_points.txt").exists(): + _fail("entry_points_present") + distribution = importlib.metadata.PathDistribution(dist_info) + if ( + list(distribution.entry_points) + or distribution.metadata["Name"] != _DISTRIBUTION_NAME + or distribution.version != _DISTRIBUTION_VERSION + ): + _fail("entry_points_present") + selected_paths = { + relative_launcher, + _BOOTSTRAP_NAME, + "netveil/__init__.py", + "netveil/cli.py", + "netveil/model.py", + "netveil/parser.py", + "netveil/privacy.py", + "netveil/py.typed", + f"{_DIST_INFO}/METADATA", + record_record_path, + f"{_DIST_INFO}/WHEEL", + f"{_DIST_INFO}/licenses/LICENSE", + f"{_DIST_INFO}/top_level.txt", + } + selected_rows = tuple( + InstalledRecordRowEvidence( + path=path, + sha256=_record_sha256_hex(records[path].digest), + size_bytes=records[path].size, + ) + for path in sorted(selected_paths & records.keys()) + ) + if not { + relative_launcher, + record_record_path, + }.issubset(row.path for row in selected_rows): + _fail("record_invalid") + evidence = InstalledEvidence( + launcher=InstalledFileEvidence( + logical_path="bin/netveil-audit", + mode=f"{stat.S_IMODE(launcher_status.st_mode):04o}", + sha256=hashlib.sha256(launcher_payload).hexdigest(), + size_bytes=len(launcher_payload), + ), + record=InstalledFileEvidence( + logical_path=f"site-packages/{record_record_path}", + mode=f"{stat.S_IMODE(record_status.st_mode):04o}", + sha256=hashlib.sha256(record_payload).hexdigest(), + size_bytes=len(record_payload), + ), + selected_record_rows=selected_rows, + ) + return InstalledLayout( + prefix=prefix, + python=python, + launcher=launcher, + site_root=site_root, + dist_info=dist_info, + bootstrap=bootstrap, + package_root=package_root, + record=record_path, + evidence=evidence, + ) + + +def _assert_no_private_output( + result: ProcessResult, + forbidden: Sequence[bytes], +) -> None: + output = result.stdout + result.stderr + if any(token and token in output for token in forbidden): + _fail("private_output_detected") + + +def _invoke( + layout: InstalledLayout, + arguments: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str], + through_path: bool = False, +) -> ProcessResult: + command = _LAUNCHER_NAME if through_path else str(layout.launcher) + return _run_process((command, *arguments), cwd=cwd, env=env) + + +def _verify_user_commands( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> None: + forbidden = (str(root).encode(),) + expected_version = b"netveil-audit 0.3.0\n" + for arguments, exact_stdout in ( + (("--version",), expected_version), + (("--help",), None), + ): + direct = _invoke(layout, arguments, cwd=root, env=env) + path = _invoke( + layout, + arguments, + cwd=root, + env=env, + through_path=True, + ) + if ( + direct.returncode != 0 + or path.returncode != 0 + or direct.stderr + or path.stderr + or direct.stdout != path.stdout + or (exact_stdout is not None and direct.stdout != exact_stdout) + ): + _fail("command_contract_failed") + if exact_stdout is None and ( + not direct.stdout.startswith(b"usage: netveil-audit ") + or b"receipt" not in direct.stdout + or b"--version" not in direct.stdout + ): + _fail("help_contract_failed") + _assert_no_private_output(direct, forbidden) + _assert_no_private_output(path, forbidden) + + +def _updated_record( + payload: bytes, + *, + path: str, + replacement: bytes, +) -> bytes: + records = _parse_record(payload) + if path not in records: + _fail("record_rewrite_failed") + try: + rows = list(csv.reader(io.StringIO(payload.decode("utf-8"), newline=""))) + except (UnicodeError, csv.Error): + _fail("record_rewrite_failed") + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + replaced = 0 + for row in rows: + if row[0] == path: + writer.writerow( + ( + path, + f"sha256={_record_digest(replacement)}", + str(len(replacement)), + ) + ) + replaced += 1 + else: + writer.writerow(row) + if replaced != 1: + _fail("record_rewrite_failed") + return output.getvalue().encode("utf-8") + + +@contextmanager +def _temporary_bytes(path: Path, payload: bytes) -> Iterator[None]: + existed = path.exists() + original = b"" + original_mode = 0 + if existed: + try: + status = path.lstat() + if not stat.S_ISREG(status.st_mode): + _fail("mutation_target_invalid") + original = path.read_bytes() + original_mode = stat.S_IMODE(status.st_mode) + except OSError: + _fail("mutation_target_invalid") + try: + path.write_bytes(payload) + yield + except OSError: + _fail("mutation_failed") + finally: + try: + if existed: + path.write_bytes(original) + path.chmod(original_mode) + else: + path.unlink(missing_ok=True) + except OSError: + _fail("mutation_restore_failed") + + +def _poisoned_environment( + env: Mapping[str, str], + *, + attack_root: Path, + marker: Path, +) -> dict[str, str]: + poisoned = dict(env) + poisoned.update( + { + "NETVEIL_VERIFY_MARKER": str(marker), + "PYTHONHOME": str(attack_root / "invalid-python-home"), + "PYTHONINSPECT": "1", + "PYTHONPATH": str(attack_root), + "PYTHONSTARTUP": str(attack_root / "startup.py"), + } + ) + return poisoned + + +def _verify_isolation( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> InterpreterEvidence: + attack_root = root / "startup-attack" + attack_root.mkdir() + marker = root / "startup-marker" + poisoned = _poisoned_environment(env, attack_root=attack_root, marker=marker) + attack_paths = ( + attack_root / "sitecustomize.py", + attack_root / "usercustomize.py", + attack_root / "startup.py", + layout.site_root / "sitecustomize.py", + layout.site_root / "usercustomize.py", + ) + try: + for path in attack_paths: + if path.exists(): + _fail("startup_probe_collision") + path.write_bytes(_ATTACK_SOURCE) + for through_path in (False, True): + result = _invoke( + layout, + ("--version",), + cwd=root, + env=poisoned, + through_path=through_path, + ) + if ( + result.returncode != 0 + or result.stdout != b"netveil-audit 0.3.0\n" + or result.stderr + or marker.exists() + ): + _fail("environment_isolation_failed") + _assert_no_private_output( + result, + (str(root).encode(), str(marker).encode()), + ) + + original_record = layout.record.read_bytes() + rewritten = _updated_record( + original_record, + path=_BOOTSTRAP_NAME, + replacement=_PROBE_SOURCE, + ) + with ( + _temporary_bytes(layout.bootstrap, _PROBE_SOURCE), + _temporary_bytes(layout.record, rewritten), + ): + probe = _invoke( + layout, + ("isolation-probe",), + cwd=root, + env=poisoned, + ) + try: + probe_document = json.loads(probe.stdout) + except (UnicodeError, json.JSONDecodeError): + _fail("isolation_probe_failed") + if not isinstance(probe_document, dict): + _fail("isolation_probe_failed") + implementation = probe_document.pop("implementation", None) + version = probe_document.pop("version", None) + cache_tag = probe_document.pop("cache_tag", None) + expected_version = ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + ) + if ( + probe.returncode != 0 + or probe.stderr + or probe_document != _EXPECTED_PROBE + or implementation != sys.implementation.name + or version != expected_version + or cache_tag != sys.implementation.cache_tag + or marker.exists() + ): + _fail("isolation_probe_failed") + _assert_no_private_output( + probe, + (str(root).encode(), str(marker).encode()), + ) + if ( + not isinstance(implementation, str) + or not isinstance(version, str) + or not isinstance(cache_tag, str) + ): + _fail("isolation_probe_failed") + evidence = InterpreterEvidence( + implementation=implementation, + version=version, + cache_tag=cache_tag, + ) + finally: + for path in attack_paths: + try: + path.unlink(missing_ok=True) + except OSError: + _fail("startup_probe_cleanup_failed") + return evidence + + +def _unchecked_hash_pyc(source: bytes, claimed_source: bytes) -> bytes: + code = compile(source, "", "exec", dont_inherit=True) + if not isinstance(code, CodeType): + _fail("pyc_probe_failed") + source_hash = importlib.util.source_hash(claimed_source) + return ( + importlib.util.MAGIC_NUMBER + + struct.pack(" None: + if marker.exists(): + _fail("adversarial_code_executed") + + +def _expect_successful_version( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], + marker: Path, +) -> None: + result = _invoke(layout, ("--version",), cwd=root, env=env) + if ( + result.returncode != 0 + or result.stdout != b"netveil-audit 0.3.0\n" + or result.stderr + ): + _fail("bytecode_inertness_failed") + _marker_absent(marker) + _assert_no_private_output( + result, + (str(root).encode(), str(marker).encode()), + ) + + +def _expect_artifact_failure( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], + marker: Path, +) -> None: + marker.unlink(missing_ok=True) + result = _invoke(layout, ("--version",), cwd=root, env=env) + if result.returncode != 10 or result.stdout or result.stderr != _ARTIFACT_FAILURE: + _fail("tamper_not_rejected") + _marker_absent(marker) + _assert_no_private_output( + result, + (str(root).encode(), str(marker).encode()), + ) + + +def _verify_bytecode_and_tamper( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> None: + marker = root / "adversarial-marker" + adversarial_env = dict(env) + adversarial_env["NETVEIL_VERIFY_MARKER"] = str(marker) + cache_tag = sys.implementation.cache_tag + if not cache_tag: + _fail("pyc_probe_failed") + + top_cache = layout.site_root / "__pycache__" + package_cache = layout.package_root / "__pycache__" + top_cache.mkdir(exist_ok=True) + package_cache.mkdir(exist_ok=True) + bootstrap_source = layout.bootstrap.read_bytes() + cli_path = layout.package_root / "cli.py" + cli_source = cli_path.read_bytes() + malicious_bootstrap_pyc = _unchecked_hash_pyc( + _ATTACK_SOURCE, + bootstrap_source, + ) + malicious_cli_pyc = _unchecked_hash_pyc(_ATTACK_SOURCE, cli_source) + + top_pyc = top_cache / f"netveil_bootstrap.{cache_tag}.pyc" + with _temporary_bytes(top_pyc, malicious_bootstrap_pyc): + _expect_successful_version( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + cli_pyc = package_cache / f"cli.{cache_tag}.pyc" + with _temporary_bytes(cli_pyc, malicious_cli_pyc): + _expect_successful_version( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + unknown_pyc = package_cache / f"unknown.{cache_tag}.pyc" + with _temporary_bytes(unknown_pyc, malicious_cli_pyc): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + launcher_payload = layout.launcher.read_bytes() + function_header = ( + b"def _execute_bootstrap(payload: bytes, bootstrap_path: Path) -> int:\n" + ) + if launcher_payload.count(function_header) != 1: + _fail("tamper_probe_unavailable") + launcher_tamper = launcher_payload.replace( + function_header, + function_header + + b' Path(os.environ["NETVEIL_VERIFY_MARKER"]).write_bytes(b"executed")\n', + ) + tamper_cases = ( + (layout.launcher, launcher_tamper), + (layout.bootstrap, bootstrap_source + b"\n" + _ATTACK_SOURCE), + (cli_path, cli_source + b"\n" + _ATTACK_SOURCE), + ) + for path, payload in tamper_cases: + with _temporary_bytes(path, payload): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + metadata_paths = ( + layout.dist_info / "METADATA", + layout.dist_info / "WHEEL", + layout.dist_info / "top_level.txt", + ) + for path in metadata_paths: + original = path.read_bytes() + with _temporary_bytes(path, original + b"\n# uncoordinated drift\n"): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + record_payload = layout.record.read_bytes() + record_drift = _updated_record( + record_payload, + path=_BOOTSTRAP_NAME, + replacement=b"uncoordinated bootstrap claim", + ) + with _temporary_bytes(layout.record, record_drift): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + unknown_source = layout.package_root / "unknown.py" + if unknown_source.exists(): + _fail("unknown_file_probe_collision") + with _temporary_bytes(unknown_source, _ATTACK_SOURCE): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + _expect_successful_version( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + +def _canonical_json(document: object) -> bytes: + return json.dumps( + document, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def _verify_receipt_document( + payload: bytes, + *, + key: bytes, +) -> dict[str, object]: + if not payload.endswith(b"\n") or payload.endswith(b"\n\n"): + _fail("receipt_not_canonical") + try: + document = json.loads(payload) + except (UnicodeError, json.JSONDecodeError): + _fail("receipt_invalid") + if ( + not isinstance(document, dict) + or document.get("schema") != "netveil.aggregate-receipt.v1" + or _canonical_json(document) + b"\n" != payload + ): + _fail("receipt_not_canonical") + report = document.get("report") + digest = document.get("report_digest") + if not isinstance(report, dict) or not isinstance(digest, dict): + _fail("receipt_invalid") + counts = report.get("counts") + by_version = report.get("endpoint_occurrences_by_ip_version") + by_scope = report.get("endpoint_occurrences_by_scope") + duplicates = report.get("duplicates") + if ( + counts + != { + "endpoint_occurrences": 5, + "physical_lines": 6, + "source_bytes": len(_CORPUS), + "unique_endpoints": 4, + } + or not isinstance(by_version, dict) + or by_version.get("ipv4") != 4 + or by_version.get("ipv6") != 1 + or not isinstance(by_scope, dict) + or by_scope.get("documentation") != 5 + or not isinstance(duplicates, dict) + or duplicates.get("group_count") != 1 + ): + _fail("receipt_semantics_invalid") + expected_digest = hashlib.sha256(_canonical_json(report)).hexdigest() + if digest != {"algorithm": "sha256", "value": expected_digest}: + _fail("receipt_digest_invalid") + source_id = report.get("source_content_id") + if ( + not isinstance(source_id, str) + or not source_id.startswith("nvs1_") + or len(source_id) != 69 + ): + _fail("receipt_identifier_invalid") + + forbidden = ( + *_RAW_ENDPOINT_TOKENS, + hashlib.sha256(_CORPUS).hexdigest().encode(), + key, + key.hex().encode(), + base64.b64encode(key), + base64.urlsafe_b64encode(key), + ) + if any(token and token in payload for token in forbidden): + _fail("receipt_private_data_detected") + return dict(document) + + +def _verify_receipt( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> ReceiptEvidence: + corpus_path = root / "documentation-corpus.txt" + key_path = root / "private-receipt.key" + _write_exclusive(corpus_path, _CORPUS, mode=0o600) + key = os.urandom(32) + _write_exclusive(key_path, key, mode=0o600) + key_status = key_path.lstat() + if ( + not stat.S_ISREG(key_status.st_mode) + or stat.S_IMODE(key_status.st_mode) != 0o600 + or key_status.st_uid != os.geteuid() + or key_status.st_nlink != 1 + ): + _fail("private_key_policy_failed") + arguments = ("receipt", str(corpus_path), "--key-file", str(key_path)) + first = _invoke(layout, arguments, cwd=root, env=env) + second = _invoke(layout, arguments, cwd=root, env=env) + through_path = _invoke( + layout, + arguments, + cwd=root, + env=env, + through_path=True, + ) + if ( + first.returncode != 0 + or second.returncode != 0 + or through_path.returncode != 0 + or first.stderr + or second.stderr + or through_path.stderr + or first.stdout != second.stdout + or first.stdout != through_path.stdout + ): + _fail("receipt_command_failed") + _assert_no_private_output( + first, + (str(root).encode(), str(corpus_path).encode(), str(key_path).encode()), + ) + _verify_receipt_document(first.stdout, key=key) + return ReceiptEvidence( + corpus=corpus_path, + key=key_path, + output=first.stdout, + ) + + +def _capture_public_demo( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> PublicDemoEvidence: + demo_root = root / "public-demo" + try: + demo_root.mkdir(mode=0o700) + except OSError: + _fail("public_demo_failed") + corpus_path = demo_root / "documentation-corpus.txt" + key_path = demo_root / "public-demo.key" + _write_exclusive(corpus_path, _CORPUS, mode=0o600) + _write_exclusive(key_path, _PUBLIC_DEMO_KEY, mode=0o600) + try: + key_status = key_path.lstat() + except OSError: + _fail("public_demo_failed") + if ( + not stat.S_ISREG(key_status.st_mode) + or stat.S_IMODE(key_status.st_mode) != 0o600 + or key_status.st_uid != os.geteuid() + or key_status.st_nlink != 1 + ): + _fail("public_demo_failed") + + version = _invoke( + layout, + ("--version",), + cwd=demo_root, + env=env, + through_path=True, + ) + receipt = _invoke( + layout, + ("receipt", corpus_path.name, "--key-file", key_path.name), + cwd=demo_root, + env=env, + through_path=True, + ) + if ( + version.returncode != 0 + or version.stdout != b"netveil-audit 0.3.0\n" + or version.stderr + or receipt.returncode != 0 + or receipt.stderr + ): + _fail("public_demo_failed") + _assert_no_private_output( + version, + (str(root).encode(), str(corpus_path).encode(), str(key_path).encode()), + ) + _assert_no_private_output( + receipt, + (str(root).encode(), str(corpus_path).encode(), str(key_path).encode()), + ) + document = _verify_receipt_document(receipt.stdout, key=_PUBLIC_DEMO_KEY) + try: + version_stdout = version.stdout.decode("ascii") + except UnicodeError: + _fail("public_demo_failed") + return PublicDemoEvidence( + corpus_sha256=hashlib.sha256(_CORPUS).hexdigest(), + corpus_size_bytes=len(_CORPUS), + corpus_physical_lines=_CORPUS.count(b"\n"), + public_key_sha256=hashlib.sha256(_PUBLIC_DEMO_KEY).hexdigest(), + public_key_size_bytes=len(_PUBLIC_DEMO_KEY), + version_stdout=version_stdout, + receipt=document, + receipt_stdout_sha256=hashlib.sha256(receipt.stdout).hexdigest(), + ) + + +def _strace_binary() -> Path: + if not sys.platform.startswith("linux"): + _fail("syscall_trace_unsupported") + located = shutil.which( + "strace", + path="/usr/bin:/bin:/usr/sbin:/sbin", + ) + if located is None: + _fail("strace_unavailable") + path = Path(located) + try: + status = path.lstat() + except OSError: + _fail("strace_unavailable") + if ( + not stat.S_ISREG(status.st_mode) + or not status.st_mode & stat.S_IXUSR + or status.st_mode & (stat.S_ISUID | stat.S_ISGID) + ): + _fail("strace_unavailable") + return path + + +def _read_trace_files(prefix: Path) -> tuple[bytes, ...]: + paths = sorted(prefix.parent.glob(f"{prefix.name}.*")) + if not paths: + _fail("syscall_trace_missing") + payloads: list[bytes] = [] + for path in paths: + try: + status = path.lstat() + if ( + not stat.S_ISREG(status.st_mode) + or status.st_size < 1 + or status.st_size > _MAX_PROCESS_OUTPUT_BYTES + ): + _fail("syscall_trace_invalid") + payload = path.read_bytes() + except OSError: + _fail("syscall_trace_invalid") + if len(payload) != status.st_size: + _fail("syscall_trace_invalid") + payloads.append(payload) + return tuple(payloads) + + +def _validate_syscall_trace( + payloads: Sequence[bytes], + *, + layout: InstalledLayout, + label: str = "unit", +) -> TraceEvidence: + if len(payloads) != 1: + _fail("post_launch_process_detected") + try: + lines = payloads[0].decode("utf-8").splitlines() + except UnicodeError: + _fail("syscall_trace_invalid") + if not lines: + _fail("syscall_trace_invalid") + exec_paths: list[str] = [] + exit_syscall_count = 0 + for line in lines: + syscall = line.split("(", 1)[0].strip() + if syscall == "execve": + match = re.match(r'^execve\("([^"\\]+)"', line) + if match is None or not line.rstrip().endswith("= 0"): + _fail("exec_chain_invalid") + exec_paths.append(match.group(1)) + elif syscall in ("exit", "exit_group"): + exit_syscall_count += 1 + else: + # The trace selector contains only process and network syscalls. + # Anything else is therefore a fork/clone/wait or network action. + _fail("network_or_process_activity_detected") + expected = [str(layout.launcher), str(layout.python)] + if exec_paths != expected or exit_syscall_count < 1: + _fail("exec_chain_invalid") + normalized = { + "exec_chain": ["installed_launcher", "installed_python"], + "exec_count": len(exec_paths), + "exit_syscall_count": exit_syscall_count, + "label": label, + "network_syscall_count": 0, + "post_launch_process_count": 0, + "process_count": len(payloads), + } + return TraceEvidence( + label=label, + normalized_sha256=hashlib.sha256(_canonical_json(normalized)).hexdigest(), + process_count=len(payloads), + exec_chain=("installed_launcher", "installed_python"), + exec_count=len(exec_paths), + exit_syscall_count=exit_syscall_count, + network_syscall_count=0, + post_launch_process_count=0, + ) + + +def _trace_invocation( + layout: InstalledLayout, + arguments: Sequence[str], + *, + root: Path, + env: Mapping[str, str], + label: str, + expected_stdout: bytes, +) -> TraceEvidence: + prefix = root / f"syscall-trace-{label}" + result = _run_process( + ( + str(_strace_binary()), + "-ff", + "-qq", + "-s", + "256", + "-e", + "trace=network,process", + "-o", + str(prefix), + "--", + str(layout.launcher), + *arguments, + ), + cwd=root, + env=env, + ) + if result.returncode != 0 or result.stdout != expected_stdout or result.stderr: + _fail("traced_command_failed") + _assert_no_private_output(result, (str(root).encode(),)) + return _validate_syscall_trace( + _read_trace_files(prefix), + layout=layout, + label=label, + ) + + +def _verify_syscall_traces( + layout: InstalledLayout, + receipt: ReceiptEvidence, + *, + root: Path, + env: Mapping[str, str], +) -> tuple[TraceEvidence, ...]: + version = _trace_invocation( + layout, + ("--version",), + root=root, + env=env, + label="version", + expected_stdout=b"netveil-audit 0.3.0\n", + ) + receipt_trace = _trace_invocation( + layout, + ( + "receipt", + str(receipt.corpus), + "--key-file", + str(receipt.key), + ), + root=root, + env=env, + label="receipt", + expected_stdout=receipt.output, + ) + return (receipt_trace, version) + + +def _platform_evidence() -> PlatformEvidence: + try: + uname = os.uname() + except (AttributeError, OSError): + _fail("platform_unsupported") + values = (sys.platform, uname.sysname, uname.release, uname.machine) + if any( + not isinstance(value, str) + or not value + or len(value) > 256 + or any(ord(character) < 32 or ord(character) == 127 for character in value) + for value in values + ): + _fail("platform_unsupported") + return PlatformEvidence( + sys_platform=sys.platform, + system=uname.sysname, + release=uname.release, + machine=uname.machine, + ) + + +def _validate_source_commit(source_commit: str) -> None: + if len(source_commit) != 40 or any( + character not in _LOWER_HEX for character in source_commit + ): + _fail("source_commit_invalid") + + +def verify_wheel( + path: Path, + *, + inventory_path: Path, + sdist_path: Path, + source_commit: str, +) -> VerificationSummary: + """Run the complete offline fresh-wheel verification.""" + + _validate_source_commit(source_commit) + inventory = _parse_release_inventory(_read_exact_inventory(inventory_path)) + if inventory.source_commit != source_commit: + _fail("inventory_source_commit_mismatch") + wheel_payload = _read_exact_wheel(path) + sdist_payload = _read_exact_sdist(sdist_path) + _bind_release_inventory( + inventory, + expected_source_commit=source_commit, + wheel_path=path, + wheel_payload=wheel_payload, + sdist_path=sdist_path, + sdist_payload=sdist_payload, + ) + sdist_members = _inspect_sdist( + sdist_payload, + source_date_epoch=inventory.source_date_epoch, + ) + _bind_sdist_members(inventory, sdist_members) + wheel = _inspect_wheel(wheel_payload) + with tempfile.TemporaryDirectory(prefix="netveil-wheel-verifier-") as raw_root: + root = Path(raw_root) + layout = _create_fresh_install(root, wheel) + environment = _base_environment(root, layout) + _verify_user_commands(layout, root=root, env=environment) + interpreter = _verify_isolation(layout, root=root, env=environment) + _verify_bytecode_and_tamper(layout, root=root, env=environment) + receipt = _verify_receipt(layout, root=root, env=environment) + public_demo = _capture_public_demo( + layout, + root=root, + env=environment, + ) + syscall_traces = _verify_syscall_traces( + layout, + receipt, + root=root, + env=environment, + ) + final_layout = _inspect_install( + layout.prefix, + layout.python, + wheel.launcher, + ) + installed = final_layout.evidence + if installed is None: + _fail("installed_layout_invalid") + return VerificationSummary( + source_commit=source_commit, + release_inventory=inventory, + installed=installed, + interpreter=interpreter, + platform=_platform_evidence(), + syscall_traces=syscall_traces, + public_demo=public_demo, + wheel_sha256=wheel.sha256, + wheel_size_bytes=len(wheel.payload), + wheel_members=wheel.members, + ) + + +def _write_text(stream: object, payload: str) -> bool: + writer = getattr(stream, "write", None) + flusher = getattr(stream, "flush", None) + if not callable(writer) or not callable(flusher): + return False + offset = 0 + try: + while offset < len(payload): + written = writer(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return False + offset += written + flusher() + except OSError: + return False + return True + + +def main(argv: Sequence[str] | None = None) -> int: + """CLI entry point with stable, path-free output.""" + + arguments = list(sys.argv[1:] if argv is None else argv) + if arguments == ["--help"]: + text = ( + "usage: verify_fresh_wheel.py --source-commit COMMIT " + "--inventory release-inventory.json " + "--sdist netveil_audit-0.3.0.tar.gz " + "netveil_audit-0.3.0-py3-none-any.whl\n" + ) + return 0 if _write_text(sys.stdout, text) else 70 + if ( + len(arguments) != 7 + or arguments[0] != "--source-commit" + or arguments[2] != "--inventory" + or arguments[3].startswith("-") + or arguments[4] != "--sdist" + or arguments[5].startswith("-") + or arguments[6].startswith("-") + ): + _write_text(sys.stderr, "netveil-wheel-verifier: usage_error\n") + return 2 + try: + source_commit = arguments[1] + _validate_source_commit(source_commit) + summary = verify_wheel( + Path(arguments[6]), + inventory_path=Path(arguments[3]), + sdist_path=Path(arguments[5]), + source_commit=source_commit, + ) + rendered = _canonical_json(summary.document()).decode("ascii") + "\n" + return 0 if _write_text(sys.stdout, rendered) else 70 + except VerificationFailure as failure: + _write_text( + sys.stderr, + f"netveil-wheel-verifier: {failure.code}\n", + ) + return 1 + except Exception: # noqa: BLE001 - the CLI boundary must redact all internals. + _write_text(sys.stderr, "netveil-wheel-verifier: internal_error\n") + return 70 + + +if __name__ == "__main__": + raise SystemExit(main()) From 8070f634bc571c46b9934efde0c423ccd641b2ba Mon Sep 17 00:00:00 2001 From: Omar Ibrahim <31526072+omar07ibrahim@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:32:45 +0000 Subject: [PATCH 7/7] Publish reproducible release evidence --- README.md | 32 ++++++ docs/assets/artifact-provenance.svg | 59 +++++++++++ docs/assets/cli-session.svg | 43 ++++++++ docs/assets/receipt-counts.svg | 53 ++++++++++ docs/assets/verification-matrix.svg | 110 ++++++++++++++++++++ docs/evidence/cli-session.cast | 6 ++ docs/evidence/fresh-wheel-verification.json | 1 + docs/evidence/release-inventory.json | 1 + docs/evidence/visual-manifest.json | 1 + tests/test_visuals.py | 47 ++++++++- tools/render_evidence.py | 4 +- 11 files changed, 354 insertions(+), 3 deletions(-) create mode 100644 docs/assets/artifact-provenance.svg create mode 100644 docs/assets/cli-session.svg create mode 100644 docs/assets/receipt-counts.svg create mode 100644 docs/assets/verification-matrix.svg create mode 100644 docs/evidence/cli-session.cast create mode 100644 docs/evidence/fresh-wheel-verification.json create mode 100644 docs/evidence/release-inventory.json create mode 100644 docs/evidence/visual-manifest.json diff --git a/README.md b/README.md index c8034d4..64b7724 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,13 @@ Receipt disclosure is still a decision: sizes, category totals, duplicate equality, and duplicate frequency remain visible. Pseudonymized does not mean anonymous. +![Counts disclosed by the verifier-captured public receipt](docs/assets/receipt-counts.svg) + +_This chart is generated from the exact successful receipt captured by the +fresh-wheel verifier. The six-line corpus uses only synthetic IETF +documentation ranges; the chart is evidence of the workflow, not a claim about +live infrastructure._ + ## Installed workflow Use an exact wheel in a fresh standard POSIX virtual environment. Verify its @@ -88,6 +95,13 @@ bytes, effective-user ownership, exactly one hard link, and owner-only permissions. The complete file and exit-code contract is in [docs/cli-contract.md](docs/cli-contract.md). +![Exact stdout captured from the installed wheel](docs/assets/cli-session.svg) + +_This is a deterministic rendering of actual stdout from the verified wheel, +not a typed mockup. The same capture is committed as an +[asciinema v2 terminal recording](docs/evidence/cli-session.cast); replay it +with `asciinema play docs/evidence/cli-session.cast`._ + ## How the guard works ![Code-derived Netveil installed execution boundary](docs/assets/architecture.svg) @@ -212,6 +226,8 @@ determinism, and redaction cases. ## Reproducible release evidence +![Source-to-execution artifact provenance](docs/assets/artifact-provenance.svg) + The publication gate requires Git, CPython 3.12, the pinned development packages above, and Linux `strace`. Run it only from a clean checkout. The builder enforces a clean Git top-level, exports the exact `HEAD` with @@ -244,6 +260,22 @@ the pinned wheel into a fresh no-dependency environment; runs startup, bytecode, tamper, deterministic-output, redaction, public-demo, and syscall checks; and emits one path-free canonical JSON line. +![Fresh-wheel verification matrix](docs/assets/verification-matrix.svg) + +The committed snapshot records exact source commit +`f3e4db737c4fd87b4b58f842a926ebb0f9889e69`: all 19 verifier checks passed, +both normalized Linux traces recorded zero network syscalls and zero +post-launch child processes, and the installed launcher and `RECORD` identities +were captured. Inspect the canonical +[release inventory](docs/evidence/release-inventory.json), the [fresh-wheel +verification record](docs/evidence/fresh-wheel-verification.json), and the +[visual manifest](docs/evidence/visual-manifest.json) rather than trusting the +rendered summary alone. + +`docs/evidence` is deliberately repository-only. Embedding an inventory that +hashes the sdist inside that same sdist would be self-referential; publish the +evidence beside the release artifacts, not inside them. + To refresh the committed evidence views after a successful exact-commit run: ```bash diff --git a/docs/assets/artifact-provenance.svg b/docs/assets/artifact-provenance.svg new file mode 100644 index 0000000..ea93381 --- /dev/null +++ b/docs/assets/artifact-provenance.svg @@ -0,0 +1,59 @@ + +Netveil release evidence chain +Source commit, artifact inventory, installed-file, and fresh execution evidence with an explicit unsigned non-claim. + + +Release evidence chain +Every displayed value comes from the canonical release inventory or fresh-wheel verifier JSON. + + +1 · clean Git HEAD +f3e4db737c4fd87b4b58f842a926ebb0f9889e69 +SOURCE_DATE_EPOCH 1785338401 +BUILDER-OBSERVED + + + + +2 · bound release artifacts +wheel · 25.8 KiB · 0f4254362623a344e247… +sdist · 105.3 KiB · 51 members +BOUND IN EVIDENCE + + + + +3 · fresh installed state +launcher 0711 · 8.5 KiB +RECORD 0600 · 13 selected rows +BOUND IN EVIDENCE + + + + +4 · executed evidence +19 checks pass +2 traces · network 0 · children 0 +BOUND IN EVIDENCE + +Exact installed bindings +launcher sha256 056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac +RECORD sha256 88c0fe5e9468c9b99aeac359f8a7b71df99b546e4ba73cdce8561cfe5fc002e0 +inventory sha256 e41685491e63cdfb8860ec7cce48ca1f8743da6d29cc7b5d3c398f738c1c6622 + +Claim boundary +The manifest is unsigned. It shows internally consistent bytes and an executed test run; it does not prove publisher identity, an honest host, or an unmodified verifier. +Authenticate the published artifact digests through a separate trusted channel. + diff --git a/docs/assets/cli-session.svg b/docs/assets/cli-session.svg new file mode 100644 index 0000000..7df6811 --- /dev/null +++ b/docs/assets/cli-session.svg @@ -0,0 +1,43 @@ + +Netveil installed CLI evidence +Terminal rendering of exact version and receipt standard output captured from the verified fresh wheel. + + +Installed CLI · captured stdout +Exact successful output captured by the fresh-wheel verifier; long canonical JSON is visually wrapped only. + + + + +Netveil installed-wheel public demonstration +source commit: f3e4db737c4fd87b4b58f842a926ebb0f9889e69 +input class: synthetic IETF documentation ranges +key class: public non-secret test material + +$ netveil-audit --version +netveil-audit 0.3.0 +$ netveil-audit receipt documentation-corpus.txt --key-file public-demo.key +{"canonicalization":"netveil.sorted-keys-json.v1","protocol":"netveil.hmac-sha256-pseudonymization.v1","report":{"canonicaliza + tion":"netveil.sorted-keys-json.v1","counts":{"endpoint_occurrences":5,"physical_lines":6,"source_bytes":129,"unique_endpoin + ts":4},"duplicates":{"extra_occurrences":1,"group_count":1,"groups":[{"extra_occurrences":1,"id":"nvd1_f382d197a21771fcdbc8d + a8d818d4612a6a6bf7d8ca45415298f475b5709da10","occurrences":2}]},"endpoint_occurrences_by_ip_version":{"ipv4":4,"ipv6":1},"en + dpoint_occurrences_by_port_bucket":{"dynamic_49152_65535":1,"registered_1024_49151":1,"system_1_1023":3},"endpoint_occurrenc + es_by_scope":{"documentation":5,"global":0,"link_local":0,"loopback":0,"multicast":0,"private":0,"reserved":0,"shared":0,"si + te_local":0,"unspecified":0},"protocol":"netveil.hmac-sha256-pseudonymization.v1","runtime":{"endpoint_semantics":"python-st + dlib-ipaddress","python_implementation":"cpython","python_version":"3.12.3","schema":"netveil.python-runtime.v1"},"schema":" + netveil.aggregate-report.v1","source_content_id":"nvs1_db2e7cafc475293724cdaa15fd267fd750a46f33ea199091a14cfa7635523e31"},"r + eport_digest":{"algorithm":"sha256","value":"e266ff9c6b81df76e9224772d67958be75b90b9a8ac67161182cf39985b76f1e"},"schema":"ne + tveil.aggregate-receipt.v1"} + diff --git a/docs/assets/receipt-counts.svg b/docs/assets/receipt-counts.svg new file mode 100644 index 0000000..659c670 --- /dev/null +++ b/docs/assets/receipt-counts.svg @@ -0,0 +1,53 @@ + +Netveil public receipt count evidence +Bar chart of physical lines, endpoint counts, IP versions, and duplicate groups parsed from the committed public demo receipt. + + +Public receipt · disclosed counts +Values parsed from the exact verifier-captured receipt; identifiers and raw endpoints are not used in this chart. + +physical lines + + +6 +endpoint occurrences + + +5 +unique endpoints + + +4 +IPv4 occurrences + + +4 +IPv6 occurrences + + +1 +duplicate groups + + +1 + +Scope result +5 / 5 occurrences · documentation ranges +Synthetic corpus only; this is workflow evidence, not a claim about live infrastructure. + +Disclosure boundary +129 source bytes are summarized, not embedded +Counts, equality and frequency remain public; pseudonymized does not mean anonymous. + diff --git a/docs/assets/verification-matrix.svg b/docs/assets/verification-matrix.svg new file mode 100644 index 0000000..3ecfdcf --- /dev/null +++ b/docs/assets/verification-matrix.svg @@ -0,0 +1,110 @@ + +Netveil fresh-wheel verification matrix +All verifier pass labels, interpreter and platform identity, and normalized path-free process and network trace evidence. + + +Fresh-wheel verification matrix +19 exact checks reported pass by the committed verifier JSON. + +cpython 3.12.3 · cpython-312 +Linux 6.17.0-1019-aws · x86_64 +f3e4db737c4fd87b4b58f842a926ebb0f9889e69 +unsigned consistency evidence · not attestation + + +wheel archive +PASS + + +fresh install +PASS + + +entry points absent +PASS + + +launcher identity +PASS + + +direct and path commands +PASS + + +isolated startup +PASS + + +environment injection inert +PASS + + +unchecked bytecode inert +PASS + + +unknown bytecode rejected +PASS + + +tamper fail closed +PASS + + +metadata drift rejected +PASS + + +coordinated bootstrap record mutation accepted +PASS + + +unknown package file rejected +PASS + + +receipt deterministic +PASS + + +receipt redacted +PASS + + +public demo capture +PASS + + +syscall trace offline +PASS + + +release inventory integrity +PASS + + +source commit bound +PASS +Normalized offline syscall evidence + +receipt +exec=2 · network=0 · post-launch children=0 +normalized sha256 0cdfc79931d31b9665f522437c58fb2df4b7a30cfdf69fccec996a206b35721a + +version +exec=2 · network=0 · post-launch children=0 +normalized sha256 f49c6be35c867adf767a07fcd01788478f57ecfddee88550f992f9275d3d7b57 + diff --git a/docs/evidence/cli-session.cast b/docs/evidence/cli-session.cast new file mode 100644 index 0000000..b633f91 --- /dev/null +++ b/docs/evidence/cli-session.cast @@ -0,0 +1,6 @@ +{"env":{"SHELL":"/bin/sh","TERM":"xterm-256color"},"height":40,"timestamp":1785338401,"version":2,"width":120} +[0.1,"o","Netveil verified public demo\r\nsynthetic IETF documentation ranges \u00b7 public non-secret key\r\n\r\n"] +[0.6,"o","$ netveil-audit --version\r\n"] +[0.9,"o","netveil-audit 0.3.0\r\n"] +[1.4,"o","$ netveil-audit receipt documentation-corpus.txt --key-file public-demo.key\r\n"] +[1.8,"o","{\"canonicalization\":\"netveil.sorted-keys-json.v1\",\"protocol\":\"netveil.hmac-sha256-pseudonymization.v1\",\"report\":{\"canonicalization\":\"netveil.sorted-keys-json.v1\",\"counts\":{\"endpoint_occurrences\":5,\"physical_lines\":6,\"source_bytes\":129,\"unique_endpoints\":4},\"duplicates\":{\"extra_occurrences\":1,\"group_count\":1,\"groups\":[{\"extra_occurrences\":1,\"id\":\"nvd1_f382d197a21771fcdbc8da8d818d4612a6a6bf7d8ca45415298f475b5709da10\",\"occurrences\":2}]},\"endpoint_occurrences_by_ip_version\":{\"ipv4\":4,\"ipv6\":1},\"endpoint_occurrences_by_port_bucket\":{\"dynamic_49152_65535\":1,\"registered_1024_49151\":1,\"system_1_1023\":3},\"endpoint_occurrences_by_scope\":{\"documentation\":5,\"global\":0,\"link_local\":0,\"loopback\":0,\"multicast\":0,\"private\":0,\"reserved\":0,\"shared\":0,\"site_local\":0,\"unspecified\":0},\"protocol\":\"netveil.hmac-sha256-pseudonymization.v1\",\"runtime\":{\"endpoint_semantics\":\"python-stdlib-ipaddress\",\"python_implementation\":\"cpython\",\"python_version\":\"3.12.3\",\"schema\":\"netveil.python-runtime.v1\"},\"schema\":\"netveil.aggregate-report.v1\",\"source_content_id\":\"nvs1_db2e7cafc475293724cdaa15fd267fd750a46f33ea199091a14cfa7635523e31\"},\"report_digest\":{\"algorithm\":\"sha256\",\"value\":\"e266ff9c6b81df76e9224772d67958be75b90b9a8ac67161182cf39985b76f1e\"},\"schema\":\"netveil.aggregate-receipt.v1\"}\r\n"] diff --git a/docs/evidence/fresh-wheel-verification.json b/docs/evidence/fresh-wheel-verification.json new file mode 100644 index 0000000..4a18f6a --- /dev/null +++ b/docs/evidence/fresh-wheel-verification.json @@ -0,0 +1 @@ +{"checks":[{"name":"wheel_archive","status":"pass"},{"name":"fresh_install","status":"pass"},{"name":"entry_points_absent","status":"pass"},{"name":"launcher_identity","status":"pass"},{"name":"direct_and_path_commands","status":"pass"},{"name":"isolated_startup","status":"pass"},{"name":"environment_injection_inert","status":"pass"},{"name":"unchecked_bytecode_inert","status":"pass"},{"name":"unknown_bytecode_rejected","status":"pass"},{"name":"tamper_fail_closed","status":"pass"},{"name":"metadata_drift_rejected","status":"pass"},{"name":"coordinated_bootstrap_record_mutation_accepted","status":"pass"},{"name":"unknown_package_file_rejected","status":"pass"},{"name":"receipt_deterministic","status":"pass"},{"name":"receipt_redacted","status":"pass"},{"name":"public_demo_capture","status":"pass"},{"name":"syscall_trace_offline","status":"pass"},{"name":"release_inventory_integrity","status":"pass"},{"name":"source_commit_bound","status":"pass"}],"installed":{"launcher":{"logical_path":"bin/netveil-audit","mode":"0711","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},"record":{"logical_path":"site-packages/netveil_audit-0.3.0.dist-info/RECORD","mode":"0600","sha256":"88c0fe5e9468c9b99aeac359f8a7b71df99b546e4ba73cdce8561cfe5fc002e0","size_bytes":1609},"selected_record_rows":[{"path":"../../../bin/netveil-audit","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},{"path":"netveil/__init__.py","sha256":"ce9ae0bfd8d5080d58e86c661ddfe910d86894cd99ef7aca58053c4fdc5cfb6e","size_bytes":1216},{"path":"netveil/cli.py","sha256":"577c91cf2a11aae46273e3ebe23ecab1f6e9834261de5b914d1644ac1552e243","size_bytes":11078},{"path":"netveil/model.py","sha256":"1f333d4152ad8b956c971595300b49984a56dc9b9dd90e8cd32157f3fdffc4f3","size_bytes":6026},{"path":"netveil/parser.py","sha256":"ee9c20174454cf8029432543d4358a71d77da0f837b13a3b375ef9d2d7179e8a","size_bytes":5106},{"path":"netveil/privacy.py","sha256":"fba03e6d5a7d618797a0654252394af04f88069ddc5da13ada34199d62b5cbfd","size_bytes":18224},{"path":"netveil/py.typed","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"path":"netveil_audit-0.3.0.dist-info/METADATA","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"path":"netveil_audit-0.3.0.dist-info/RECORD","sha256":null,"size_bytes":null},{"path":"netveil_audit-0.3.0.dist-info/WHEEL","sha256":"2b6eb4118ce7cd7b09601406aa623c553c4476265836f0d9c16f5c061f7efcc0","size_bytes":91},{"path":"netveil_audit-0.3.0.dist-info/licenses/LICENSE","sha256":"125e5c962ef12e9a92e39f9af7776d8010a98fce5f4559221be7033573c226a8","size_bytes":1069},{"path":"netveil_audit-0.3.0.dist-info/top_level.txt","sha256":"9f7e05afb0b8dbbd49796a869258dc8cdc13dae0973dd848872799554d018ec7","size_bytes":26},{"path":"netveil_bootstrap.py","sha256":"71205f05f3398b7ddc6ccf4b820c001f71f2932694f370dfccee2d31a33f9de8","size_bytes":16752}]},"integrity_evidence":{"artifacts":[{"filename":"netveil_audit-0.3.0-py3-none-any.whl","kind":"wheel","sha256":"0f4254362623a344e2474db96be81534f502e291b990fcf9ab1f0ba46d0f0675","size_bytes":26440},{"filename":"netveil_audit-0.3.0.tar.gz","kind":"sdist","members":[{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/LICENSE","sha256":"125e5c962ef12e9a92e39f9af7776d8010a98fce5f4559221be7033573c226a8","size_bytes":1069},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/MANIFEST.in","sha256":"8fab5c0ebe83e4fe3ce4118045e2cbe48c70b4b60aabaadb9d06d335081f01d3","size_bytes":430},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/PKG-INFO","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/README.md","sha256":"97eb7a6405c8f2360d147fab133fab614e5fa661086a0b75bc3df2a7f62401b6","size_bytes":10824},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/SECURITY.md","sha256":"2e4f7ddfaf851ca804f2d9b6306cdda309646aa0dd87ed1f33ffec9745121201","size_bytes":8028},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/docs","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/artifact-boundary.md","sha256":"91b2026791ce76c3ab1662b79ea59649f9d6d31ab07aca7e0b4f80ba2c0361d5","size_bytes":8545},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/docs/assets","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/assets/architecture.svg","sha256":"5adc28a9088dc6f000ce810511726fe6d14cfcb6996fbff76a6e3f9154f28abc","size_bytes":8558},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/cli-contract.md","sha256":"6a1b32fed2e7fee8d0c44a1b2a1f70cb0153b63488ec9e7ca04371160484a59d","size_bytes":6541},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/privacy-protocol.md","sha256":"d047daf39cb49b69ff9c486aa0a7e9e06e9d872eb8cadfa5e13eb3bd9e2804f1","size_bytes":5213},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/pyproject.toml","sha256":"af21a11f116f31b5f0783934e1583ac1984f4c8f7dea0512812168ae47248299","size_bytes":1445},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/requirements-dev.txt","sha256":"6ca7f1739fb5f044210252ca5e6893bc1fee5ceb30130aaa44731738d253cef4","size_bytes":348},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/scripts","size_bytes":0},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/scripts/netveil-audit","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/setup.cfg","sha256":"1c473cbaee8da5fc46e7f0158794af5cea4414c34a3cf3f180c2001f5e38bd3e","size_bytes":38},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src","size_bytes":0},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src/netveil","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/__init__.py","sha256":"ce9ae0bfd8d5080d58e86c661ddfe910d86894cd99ef7aca58053c4fdc5cfb6e","size_bytes":1216},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/cli.py","sha256":"577c91cf2a11aae46273e3ebe23ecab1f6e9834261de5b914d1644ac1552e243","size_bytes":11078},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/model.py","sha256":"1f333d4152ad8b956c971595300b49984a56dc9b9dd90e8cd32157f3fdffc4f3","size_bytes":6026},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/parser.py","sha256":"ee9c20174454cf8029432543d4358a71d77da0f837b13a3b375ef9d2d7179e8a","size_bytes":5106},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/privacy.py","sha256":"fba03e6d5a7d618797a0654252394af04f88069ddc5da13ada34199d62b5cbfd","size_bytes":18224},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/py.typed","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/PKG-INFO","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/SOURCES.txt","sha256":"97e233d844643aa7fa463361c581cf46e4a6c6d0cc296c51758f8431d30ac231","size_bytes":958},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/dependency_links.txt","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/top_level.txt","sha256":"9f7e05afb0b8dbbd49796a869258dc8cdc13dae0973dd848872799554d018ec7","size_bytes":26},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_bootstrap.py","sha256":"71205f05f3398b7ddc6ccf4b820c001f71f2932694f370dfccee2d31a33f9de8","size_bytes":16752},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tests","size_bytes":0},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tests/fixtures","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/fixtures/launcher_bootstrap_fixture.py","sha256":"ce44795d98e969bd625373e9e84064c013470d641e009989e569e68d2ef95b73","size_bytes":251},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_bootstrap.py","sha256":"9b9f18cff5cc8a3499a6617861e4e8e7817ecba8495637a9f663afc95e971ea0","size_bytes":40030},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_cli.py","sha256":"8f8ea9b0617262ef74dcd148f5933f39d74b067be4b54eefff3f438b4b68308e","size_bytes":27081},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_evidence_renderer.py","sha256":"76b4f6d9b4b9c350a1fcbdb32e7985ff64a2fc40c3c1f15622615235656b9c8c","size_bytes":11137},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_fresh_wheel_verifier.py","sha256":"0ed6c2d4f8c8ee65c5731005c7481b7c44655ae3fc8d9063fc455152e1f092ff","size_bytes":32914},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_launcher.py","sha256":"f9931cb6ec9e43ffc55ea9e89ee2188a3edede24df351ecd3c793a649a1f202e","size_bytes":21478},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_model.py","sha256":"fb54ea5687906b0410c0623626b2a16bc6b2b634bef17bb402811986b921b387","size_bytes":4737},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_packaging.py","sha256":"75eb1770f14b15c2baa4d8e1c3551c122d0de365a797d3849f4690e82de719d5","size_bytes":2603},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_parser.py","sha256":"bc606c4b0a7cb2c5fce74cba4133f9c625ea412c0d42670605e14e755d7305ab","size_bytes":10625},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_privacy.py","sha256":"d770c688d2dd13d0078c679dce4e2095c7a9c970a8e3c9a1d1194a7a779ad4f1","size_bytes":23160},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_release_builder.py","sha256":"b93c280b25f056c71d3c8038decc91185310d28f4af7bfb8a429b1f7c915a38e","size_bytes":20067},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_visuals.py","sha256":"5ec195c534d085b4ca59a578a54e177255da35d01a6deac419325dd786023df5","size_bytes":721},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tools","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tools/__init__.py","sha256":"28db1b0a84cbcf8a228da02aae487a38895d51b66e2ea37067058252351b23db","size_bytes":55},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/build_release.py","sha256":"979ca44ad2121588c4ddf72d721f4d5d49449546a8f5a1f9e06796d39d9e820d","size_bytes":25873},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/render_architecture.py","sha256":"25f308aa838397bf3e331351ca6781267b49b43069e066d78ad4a3acf34ae741","size_bytes":9738},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/render_evidence.py","sha256":"9b00b1a58e61d92b899d86d9777f720a8565aafdfa42b25d34e2aa505d068172","size_bytes":42639},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/verify_fresh_wheel.py","sha256":"c88109e00821895446ff4ff71bef8a283af0c70fa723e9e2a7ed630ea369f6f6","size_bytes":79572}],"sha256":"7d1975194f8a439539f10e50f71b4f37da1dc335569d941e4f25990d4cb14139","size_bytes":107816}],"attestation_verified":false,"inventory_schema":"netveil.release-inventory.v1","inventory_sha256":"e41685491e63cdfb8860ec7cce48ca1f8743da6d29cc7b5d3c398f738c1c6622","inventory_type":"unsigned_sha256_manifest","signature_verified":false,"source_commit":"f3e4db737c4fd87b4b58f842a926ebb0f9889e69","source_date_epoch":1785338401},"interpreter":{"cache_tag":"cpython-312","implementation":"cpython","version":"3.12.3"},"platform":{"machine":"x86_64","release":"6.17.0-1019-aws","sys_platform":"linux","system":"Linux"},"public_demo":{"classification":"synthetic_ietf_documentation_ranges_with_public_demo_key","commands":[{"argv":["netveil-audit","--version"],"exit_code":0,"stderr":"","stdout":"netveil-audit 0.3.0\n"},{"argv":["netveil-audit","receipt","documentation-corpus.txt","--key-file","public-demo.key"],"exit_code":0,"stderr":"","stdout_json":{"canonicalization":"netveil.sorted-keys-json.v1","protocol":"netveil.hmac-sha256-pseudonymization.v1","report":{"canonicalization":"netveil.sorted-keys-json.v1","counts":{"endpoint_occurrences":5,"physical_lines":6,"source_bytes":129,"unique_endpoints":4},"duplicates":{"extra_occurrences":1,"group_count":1,"groups":[{"extra_occurrences":1,"id":"nvd1_f382d197a21771fcdbc8da8d818d4612a6a6bf7d8ca45415298f475b5709da10","occurrences":2}]},"endpoint_occurrences_by_ip_version":{"ipv4":4,"ipv6":1},"endpoint_occurrences_by_port_bucket":{"dynamic_49152_65535":1,"registered_1024_49151":1,"system_1_1023":3},"endpoint_occurrences_by_scope":{"documentation":5,"global":0,"link_local":0,"loopback":0,"multicast":0,"private":0,"reserved":0,"shared":0,"site_local":0,"unspecified":0},"protocol":"netveil.hmac-sha256-pseudonymization.v1","runtime":{"endpoint_semantics":"python-stdlib-ipaddress","python_implementation":"cpython","python_version":"3.12.3","schema":"netveil.python-runtime.v1"},"schema":"netveil.aggregate-report.v1","source_content_id":"nvs1_db2e7cafc475293724cdaa15fd267fd750a46f33ea199091a14cfa7635523e31"},"report_digest":{"algorithm":"sha256","value":"e266ff9c6b81df76e9224772d67958be75b90b9a8ac67161182cf39985b76f1e"},"schema":"netveil.aggregate-receipt.v1"},"stdout_sha256":"de378c083090651dcc1805cc9e164ee3fa9789a9dfa7da72f921ad2cda879416"}],"corpus":{"physical_lines":6,"sha256":"49f7aab7d50edcf5a5bd2a9edc002e7c9303881817c15e8272e1e3ee1f36129e","size_bytes":129},"public_demo_key":{"classification":"public_non_secret_test_material","sha256":"2d27befbc438954c4a55d8c0e36192c5a4a7e9e3f15c17d6e427c4a3499d945d","size_bytes":32,"source_constant":"tools/verify_fresh_wheel.py:_PUBLIC_DEMO_KEY"}},"schema":"netveil.fresh-wheel-verification.v1","source_commit":"f3e4db737c4fd87b4b58f842a926ebb0f9889e69","status":"pass","syscall_traces":[{"exec_chain":["installed_launcher","installed_python"],"exec_count":2,"exit_syscall_count":1,"label":"receipt","network_syscall_count":0,"normalized_sha256":"0cdfc79931d31b9665f522437c58fb2df4b7a30cfdf69fccec996a206b35721a","post_launch_process_count":0,"process_count":1},{"exec_chain":["installed_launcher","installed_python"],"exec_count":2,"exit_syscall_count":1,"label":"version","network_syscall_count":0,"normalized_sha256":"f49c6be35c867adf767a07fcd01788478f57ecfddee88550f992f9275d3d7b57","post_launch_process_count":0,"process_count":1}],"wheel":{"members":[{"mode":"0600","path":"netveil/__init__.py","sha256":"ce9ae0bfd8d5080d58e86c661ddfe910d86894cd99ef7aca58053c4fdc5cfb6e","size_bytes":1216},{"mode":"0600","path":"netveil/cli.py","sha256":"577c91cf2a11aae46273e3ebe23ecab1f6e9834261de5b914d1644ac1552e243","size_bytes":11078},{"mode":"0600","path":"netveil/model.py","sha256":"1f333d4152ad8b956c971595300b49984a56dc9b9dd90e8cd32157f3fdffc4f3","size_bytes":6026},{"mode":"0600","path":"netveil/parser.py","sha256":"ee9c20174454cf8029432543d4358a71d77da0f837b13a3b375ef9d2d7179e8a","size_bytes":5106},{"mode":"0600","path":"netveil/privacy.py","sha256":"fba03e6d5a7d618797a0654252394af04f88069ddc5da13ada34199d62b5cbfd","size_bytes":18224},{"mode":"0644","path":"netveil/py.typed","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"mode":"0755","path":"netveil_audit-0.3.0.data/scripts/netveil-audit","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},{"mode":"0644","path":"netveil_audit-0.3.0.dist-info/METADATA","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"mode":"0664","path":"netveil_audit-0.3.0.dist-info/RECORD","sha256":"fcdee87a96f526d4d7546a733cf41b0898e917aa62b305a8b9ac637ab49df69d","size_bytes":1048},{"mode":"0600","path":"netveil_audit-0.3.0.dist-info/WHEEL","sha256":"2b6eb4118ce7cd7b09601406aa623c553c4476265836f0d9c16f5c061f7efcc0","size_bytes":91},{"mode":"0644","path":"netveil_audit-0.3.0.dist-info/licenses/LICENSE","sha256":"125e5c962ef12e9a92e39f9af7776d8010a98fce5f4559221be7033573c226a8","size_bytes":1069},{"mode":"0600","path":"netveil_audit-0.3.0.dist-info/top_level.txt","sha256":"9f7e05afb0b8dbbd49796a869258dc8cdc13dae0973dd848872799554d018ec7","size_bytes":26},{"mode":"0600","path":"netveil_bootstrap.py","sha256":"71205f05f3398b7ddc6ccf4b820c001f71f2932694f370dfccee2d31a33f9de8","size_bytes":16752}],"sha256":"0f4254362623a344e2474db96be81534f502e291b990fcf9ab1f0ba46d0f0675","size_bytes":26440}} diff --git a/docs/evidence/release-inventory.json b/docs/evidence/release-inventory.json new file mode 100644 index 0000000..13d2e9f --- /dev/null +++ b/docs/evidence/release-inventory.json @@ -0,0 +1 @@ +{"artifacts":[{"filename":"netveil_audit-0.3.0-py3-none-any.whl","kind":"wheel","sha256":"0f4254362623a344e2474db96be81534f502e291b990fcf9ab1f0ba46d0f0675","size_bytes":26440},{"filename":"netveil_audit-0.3.0.tar.gz","kind":"sdist","members":[{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/LICENSE","sha256":"125e5c962ef12e9a92e39f9af7776d8010a98fce5f4559221be7033573c226a8","size_bytes":1069},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/MANIFEST.in","sha256":"8fab5c0ebe83e4fe3ce4118045e2cbe48c70b4b60aabaadb9d06d335081f01d3","size_bytes":430},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/PKG-INFO","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/README.md","sha256":"97eb7a6405c8f2360d147fab133fab614e5fa661086a0b75bc3df2a7f62401b6","size_bytes":10824},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/SECURITY.md","sha256":"2e4f7ddfaf851ca804f2d9b6306cdda309646aa0dd87ed1f33ffec9745121201","size_bytes":8028},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/docs","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/artifact-boundary.md","sha256":"91b2026791ce76c3ab1662b79ea59649f9d6d31ab07aca7e0b4f80ba2c0361d5","size_bytes":8545},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/docs/assets","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/assets/architecture.svg","sha256":"5adc28a9088dc6f000ce810511726fe6d14cfcb6996fbff76a6e3f9154f28abc","size_bytes":8558},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/cli-contract.md","sha256":"6a1b32fed2e7fee8d0c44a1b2a1f70cb0153b63488ec9e7ca04371160484a59d","size_bytes":6541},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/privacy-protocol.md","sha256":"d047daf39cb49b69ff9c486aa0a7e9e06e9d872eb8cadfa5e13eb3bd9e2804f1","size_bytes":5213},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/pyproject.toml","sha256":"af21a11f116f31b5f0783934e1583ac1984f4c8f7dea0512812168ae47248299","size_bytes":1445},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/requirements-dev.txt","sha256":"6ca7f1739fb5f044210252ca5e6893bc1fee5ceb30130aaa44731738d253cef4","size_bytes":348},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/scripts","size_bytes":0},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/scripts/netveil-audit","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/setup.cfg","sha256":"1c473cbaee8da5fc46e7f0158794af5cea4414c34a3cf3f180c2001f5e38bd3e","size_bytes":38},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src","size_bytes":0},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src/netveil","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/__init__.py","sha256":"ce9ae0bfd8d5080d58e86c661ddfe910d86894cd99ef7aca58053c4fdc5cfb6e","size_bytes":1216},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/cli.py","sha256":"577c91cf2a11aae46273e3ebe23ecab1f6e9834261de5b914d1644ac1552e243","size_bytes":11078},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/model.py","sha256":"1f333d4152ad8b956c971595300b49984a56dc9b9dd90e8cd32157f3fdffc4f3","size_bytes":6026},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/parser.py","sha256":"ee9c20174454cf8029432543d4358a71d77da0f837b13a3b375ef9d2d7179e8a","size_bytes":5106},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/privacy.py","sha256":"fba03e6d5a7d618797a0654252394af04f88069ddc5da13ada34199d62b5cbfd","size_bytes":18224},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/py.typed","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/PKG-INFO","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/SOURCES.txt","sha256":"97e233d844643aa7fa463361c581cf46e4a6c6d0cc296c51758f8431d30ac231","size_bytes":958},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/dependency_links.txt","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/top_level.txt","sha256":"9f7e05afb0b8dbbd49796a869258dc8cdc13dae0973dd848872799554d018ec7","size_bytes":26},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_bootstrap.py","sha256":"71205f05f3398b7ddc6ccf4b820c001f71f2932694f370dfccee2d31a33f9de8","size_bytes":16752},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tests","size_bytes":0},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tests/fixtures","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/fixtures/launcher_bootstrap_fixture.py","sha256":"ce44795d98e969bd625373e9e84064c013470d641e009989e569e68d2ef95b73","size_bytes":251},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_bootstrap.py","sha256":"9b9f18cff5cc8a3499a6617861e4e8e7817ecba8495637a9f663afc95e971ea0","size_bytes":40030},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_cli.py","sha256":"8f8ea9b0617262ef74dcd148f5933f39d74b067be4b54eefff3f438b4b68308e","size_bytes":27081},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_evidence_renderer.py","sha256":"76b4f6d9b4b9c350a1fcbdb32e7985ff64a2fc40c3c1f15622615235656b9c8c","size_bytes":11137},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_fresh_wheel_verifier.py","sha256":"0ed6c2d4f8c8ee65c5731005c7481b7c44655ae3fc8d9063fc455152e1f092ff","size_bytes":32914},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_launcher.py","sha256":"f9931cb6ec9e43ffc55ea9e89ee2188a3edede24df351ecd3c793a649a1f202e","size_bytes":21478},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_model.py","sha256":"fb54ea5687906b0410c0623626b2a16bc6b2b634bef17bb402811986b921b387","size_bytes":4737},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_packaging.py","sha256":"75eb1770f14b15c2baa4d8e1c3551c122d0de365a797d3849f4690e82de719d5","size_bytes":2603},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_parser.py","sha256":"bc606c4b0a7cb2c5fce74cba4133f9c625ea412c0d42670605e14e755d7305ab","size_bytes":10625},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_privacy.py","sha256":"d770c688d2dd13d0078c679dce4e2095c7a9c970a8e3c9a1d1194a7a779ad4f1","size_bytes":23160},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_release_builder.py","sha256":"b93c280b25f056c71d3c8038decc91185310d28f4af7bfb8a429b1f7c915a38e","size_bytes":20067},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_visuals.py","sha256":"5ec195c534d085b4ca59a578a54e177255da35d01a6deac419325dd786023df5","size_bytes":721},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tools","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tools/__init__.py","sha256":"28db1b0a84cbcf8a228da02aae487a38895d51b66e2ea37067058252351b23db","size_bytes":55},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/build_release.py","sha256":"979ca44ad2121588c4ddf72d721f4d5d49449546a8f5a1f9e06796d39d9e820d","size_bytes":25873},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/render_architecture.py","sha256":"25f308aa838397bf3e331351ca6781267b49b43069e066d78ad4a3acf34ae741","size_bytes":9738},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/render_evidence.py","sha256":"9b00b1a58e61d92b899d86d9777f720a8565aafdfa42b25d34e2aa505d068172","size_bytes":42639},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/verify_fresh_wheel.py","sha256":"c88109e00821895446ff4ff71bef8a283af0c70fa723e9e2a7ed630ea369f6f6","size_bytes":79572}],"sha256":"7d1975194f8a439539f10e50f71b4f37da1dc335569d941e4f25990d4cb14139","size_bytes":107816}],"schema":"netveil.release-inventory.v1","source_commit":"f3e4db737c4fd87b4b58f842a926ebb0f9889e69","source_date_epoch":1785338401} diff --git a/docs/evidence/visual-manifest.json b/docs/evidence/visual-manifest.json new file mode 100644 index 0000000..493f4ea --- /dev/null +++ b/docs/evidence/visual-manifest.json @@ -0,0 +1 @@ +{"claim_boundary":"unsigned internally consistent build-and-execution evidence; not publisher authentication, host attestation, or a signature","generator":{"path":"tools/render_evidence.py","sha256":"1a14a0a66b1a56cd917eab0e15c87625e97a3849e47f30cd3d2985514617d3d9","size_bytes":42635},"inputs":[{"path":"docs/evidence/fresh-wheel-verification.json","sha256":"4938f49af70800713eab2c7734db149c605cad146c08940d55c7c0e0f6efc98d","size_bytes":16785},{"path":"docs/evidence/release-inventory.json","sha256":"e41685491e63cdfb8860ec7cce48ca1f8743da6d29cc7b5d3c398f738c1c6622","size_bytes":8609}],"outputs":[{"path":"docs/assets/artifact-provenance.svg","sha256":"40e22f1529c4123dfdb62322a926f36ee4206c2c6d655b8019c2ed9a1554a57a","size_bytes":4427},{"path":"docs/assets/cli-session.svg","sha256":"9a76e0d0511aa70ba7bcc703b40de62a76b93bd2459ef8bf26dd0d3f214b6120","size_bytes":4845},{"path":"docs/assets/receipt-counts.svg","sha256":"3f4d062c312f8534274f2dabd5c37bee10586e1dba34ce1cef2ba243389f1780","size_bytes":3620},{"path":"docs/assets/verification-matrix.svg","sha256":"af5bb1175f04cfd0654659e98a21f4ff3a3cb4c0525dd80f91a7e7d0735a6c62","size_bytes":7317},{"path":"docs/evidence/cli-session.cast","sha256":"4ebc2d4a493983a3490d5e0f9a04bcdaf117a2cb60ba60beadd27ad172ac8644","size_bytes":1805}],"schema":"netveil.visual-evidence.v1","source_commit":"f3e4db737c4fd87b4b58f842a926ebb0f9889e69"} diff --git a/tests/test_visuals.py b/tests/test_visuals.py index 87486f1..f781311 100644 --- a/tests/test_visuals.py +++ b/tests/test_visuals.py @@ -1,9 +1,11 @@ from __future__ import annotations +import json import unittest from pathlib import Path +from xml.etree import ElementTree -from tools import render_architecture +from tools import render_architecture, render_evidence ROOT = Path(__file__).resolve().parents[1] @@ -18,6 +20,49 @@ def test_architecture_svg_is_current_and_code_derived(self) -> None: self.assertIn("-I -E -S -B", text) self.assertIn("installed RECORD is not a signature", text) + def test_committed_release_evidence_views_are_current(self) -> None: + evidence_paths = ( + render_evidence.VERIFICATION_PATH, + render_evidence.INVENTORY_PATH, + ) + if not all(path.is_file() for path in evidence_paths): + self.assertFalse( + any(path.exists() for path in evidence_paths), + "release evidence must be either complete or absent", + ) + return + + outputs = render_evidence.render_bundle( + render_evidence.VERIFICATION_PATH.read_bytes(), + render_evidence.INVENTORY_PATH.read_bytes(), + generator_payload=render_evidence.GENERATOR_PATH.read_bytes(), + ) + for relative_path, expected in outputs.items(): + observed = (ROOT / relative_path).read_bytes() + self.assertEqual(observed, expected, relative_path) + + for relative_path in ( + render_evidence.CLI_SVG_PATH, + render_evidence.COUNTS_SVG_PATH, + render_evidence.MATRIX_SVG_PATH, + render_evidence.PROVENANCE_SVG_PATH, + ): + ElementTree.fromstring(outputs[relative_path]) + for line in outputs[render_evidence.CAST_PATH].splitlines(): + json.loads(line) + + def test_readme_presents_every_reproducible_visual(self) -> None: + readme = (ROOT / "README.md").read_text() + expected_links = { + "docs/assets/architecture.svg", + *render_evidence.VISUAL_OUTPUT_PATHS, + render_evidence.MANIFEST_PATH, + "docs/evidence/fresh-wheel-verification.json", + "docs/evidence/release-inventory.json", + } + for link in expected_links: + self.assertIn(link, readme) + if __name__ == "__main__": unittest.main() diff --git a/tools/render_evidence.py b/tools/render_evidence.py index 7e12651..ff9dded 100755 --- a/tools/render_evidence.py +++ b/tools/render_evidence.py @@ -910,7 +910,7 @@ def _render_provenance(facts: EvidenceFacts) -> bytes: ), ( 390, - "2 · deterministic artifacts", + "2 · bound release artifacts", f"wheel · {_human_bytes(wheel.size_bytes)} · {wheel.sha256[:20]}…", ( f"sdist · {_human_bytes(sdist.size_bytes)} · " @@ -1001,7 +1001,7 @@ def _render_provenance(facts: EvidenceFacts) -> bytes: return _svg_document( title="Netveil release evidence chain", description=( - "Source commit, deterministic artifact, installed-file, and " + "Source commit, artifact inventory, installed-file, and " "fresh execution evidence with an explicit unsigned non-claim." ), width=1440,