diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4ab2b56 --- /dev/null +++ b/.gitignore @@ -0,0 +1,18 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ +.venv/ +.demo-venv/ +.coverage +*.egg-info/ +dist/ +build/ + +# Raw endpoint corpora may contain third-party infrastructure and stay local. +*.endpoints +data/raw/ +corpus.txt +receipt.json +receipt.key diff --git a/5.txt b/5.txt deleted file mode 100644 index d91ea28..0000000 --- a/5.txt +++ /dev/null @@ -1,39 +0,0 @@ -62.113.115.94:16072 -72.195.114.169:4145 -72.195.34.58:4145 -72.49.49.11:31034 -5.161.93.53:1080 -192.252.214.20:15864 -208.102.51.6:58208 -70.166.167.55:57745 -72.221.196.157:35904 -67.201.33.10:25283 -72.221.172.203:4145 -218.64.255.198:7302 -193.252.50.1:3820 -67.201.33.9:25280 -150.136.189.225:80 -192.111.130.2:4145 -62.113.115.94:16072 -72.206.181.97:64943 -72.195.114.169:4145 -72.195.34.58:4145 -72.221.232.155:4145 -72.49.49.11:31034 -5.161.93.53:1080 -72.221.164.34:60671 -192.252.214.20:15864 -208.102.51.6:58208 -72.206.181.103:4145 -72.217.216.239:4145 -70.166.167.55:57745 -72.221.196.157:35904 -70.166.167.38:57728 -72.206.181.105:64935 -67.201.33.10:25283 -72.221.172.203:4145 -103.156.141.237:1080 -66.29.154.105:1080 -218.64.255.198:7302 -193.252.50.1:3820 -67.201.33.9:25280 \ No newline at end of file diff --git a/6.txt b/6.txt deleted file mode 100644 index d835d5f..0000000 --- a/6.txt +++ /dev/null @@ -1,36 +0,0 @@ -72.49.49.11:31034 -69.61.200.104:36181 -66.42.224.229:41679 -192.252.208.70:14282 -67.201.33.10:25283 -192.111.130.2:4145 -192.252.209.155:14455 -192.111.137.34:18765 -192.111.139.165:4145 -192.111.139.162:4145 -72.195.34.60:27391 -24.249.199.12:4145 -72.210.252.137:4145 -70.166.167.38:57728 -72.206.181.105:64935 -72.195.34.35:27360 -72.206.181.123:4145 -72.221.196.157:35904 -72.217.216.239:4145 -72.206.181.103:4145 -72.221.232.152:4145 -72.210.208.101:4145 -24.249.199.4:4145 -72.221.172.203:4145 -72.195.34.58:4145 -72.221.232.155:4145 -72.206.181.97:64943 -198.8.94.170:4145 -185.26.144.197:35581 -151.80.252.69:64246 -188.165.254.122:9420 -51.77.141.29:1080 -146.56.173.56:22738 -151.80.252.69:7532 -45.10.71.15:8888 -192.111.135.17:18302 \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a9918bd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Omar Ibrahim + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..63844e1 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,17 @@ +include LICENSE +include README.md +include SECURITY.md +include requirements-dev.txt +include pyproject.toml +include scripts/netveil-audit +recursive-include docs *.md +recursive-include docs/assets *.svg *.json *.cast *.gif +recursive-include src *.py py.typed +recursive-include tests *.py +recursive-include tools *.py +global-exclude *.py[cod] +global-exclude .coverage +global-exclude .DS_Store +global-exclude *~ +prune build +prune dist diff --git a/README.md b/README.md index 94d2833..64b7724 100644 --- a/README.md +++ b/README.md @@ -1 +1,317 @@ -# Hello-World \ No newline at end of file +# Netveil + +Netveil turns a private local corpus of IP endpoints into a deterministic, +pseudonymized audit receipt without resolving, connecting to, or probing any +endpoint. + +The unusual part is not the JSON. The wheel-installed command starts through +an isolated static launcher, binds selected installed files to installer- +derived distribution metadata, source-loads a closed package inventory from +already-pinned bytes, and omits raw endpoints, caller paths, key material, and +unkeyed corpus hashes from its successful receipt and Python-handled Netveil +diagnostics. + +> **Status:** 0.3.0 is a release candidate on the rehabilitation pull-request +> branch, not a published stable release. The guarded command has been +> exercised on CPython 3.12.3 / Linux x86-64. See the exact +> [artifact boundary](docs/artifact-boundary.md) before relying on it. + +## What it produces + +Input stays local: + +```text +# IETF documentation ranges only +192.0.2.10:443 +192.0.2.10:443 +[2001:db8::10]:8443 +``` + +The public receipt contains fixed-schema counts, runtime semantics, typed +HMAC-SHA256 identifiers for exact source content and repeated endpoint groups, +and a digest of the canonical public report. It does not contain +`192.0.2.10`, `2001:db8::10`, the key, or an ordinary SHA-256 of the input. + +Receipt disclosure is still a decision: sizes, category totals, duplicate +equality, and duplicate frequency remain visible. Pseudonymized does not mean +anonymous. + +![Counts disclosed by the verifier-captured public receipt](docs/assets/receipt-counts.svg) + +_This chart is generated from the exact successful receipt captured by the +fresh-wheel verifier. The six-line corpus uses only synthetic IETF +documentation ranges; the chart is evidence of the workflow, not a claim about +live infrastructure._ + +## Installed workflow + +Use an exact wheel in a fresh standard POSIX virtual environment. Verify its +published digest through an authenticated channel before installation; the +runtime guard is not a signature. + +```bash +WHEEL=dist/netveil_audit-0.3.0-py3-none-any.whl +EXPECTED_SHA256='' +printf '%s %s\n' "$EXPECTED_SHA256" "$WHEEL" \ + | sha256sum --check --strict + +python3 -m venv .demo-venv +.demo-venv/bin/python -m pip install --no-index --no-deps "$WHEEL" +.demo-venv/bin/netveil-audit --version +``` + +Create a synthetic demonstration corpus and a private ephemeral key: + +```bash +umask 077 + +printf '%s\n' \ + '# IETF documentation ranges only' \ + '192.0.2.10:443' \ + '192.0.2.10:443' \ + '[2001:db8::10]:8443' > corpus.txt + +.demo-venv/bin/python - <<'PY' +import os +import secrets + +descriptor = os.open( + "receipt.key", + os.O_WRONLY | os.O_CREAT | os.O_EXCL, + 0o600, +) +with os.fdopen(descriptor, "wb") as stream: + stream.write(secrets.token_bytes(32)) +PY + +.demo-venv/bin/netveil-audit \ + receipt corpus.txt \ + --key-file receipt.key > receipt.json +``` + +The key is exact binary input: do not create a production key with `echo`, +hard-code it, reuse a corpus as a key, or commit it. Netveil requires 32–4096 +bytes, effective-user ownership, exactly one hard link, and owner-only +permissions. The complete file and exit-code contract is in +[docs/cli-contract.md](docs/cli-contract.md). + +![Exact stdout captured from the installed wheel](docs/assets/cli-session.svg) + +_This is a deterministic rendering of actual stdout from the verified wheel, +not a typed mockup. The same capture is committed as an +[asciinema v2 terminal recording](docs/evidence/cli-session.cast); replay it +with `asciinema play docs/evidence/cli-session.cast`._ + +## How the guard works + +![Code-derived Netveil installed execution boundary](docs/assets/architecture.svg) + +_Selected launcher flags, parser/CLI bounds, and the closed module inventory +are extracted from source by `tools/render_architecture.py`; the remaining +labels document the reviewed design, and the test suite pins the complete +renderer output._ + +```text +installed polyglot launcher + │ /bin/sh -> sibling python -I -E -S -B + ▼ +startup-profile + installed-RECORD drift check + │ compile checked netveil_bootstrap.py bytes + ▼ +closed package inventory + bounded descriptor reads + │ compile pinned netveil.* source bytes in memory + ▼ +local corpus + owner-only key + │ parse -> canonicalize -> aggregate -> domain-separated HMAC + ▼ +canonical pseudonymized receipt on stdout +``` + +The static launcher closes the earlier `PYTHONPATH`, `sitecustomize`, and +unchecked-bootstrap-`pyc` gap of a generated console wrapper. The package +finder rejects unknown `netveil.*` modules and does not execute package +bytecode. + +The remaining trust boundary is explicit: + +- the installed launcher is already executing before it can check itself; +- `/bin/sh`, the sibling CPython interpreter, stdlib, installer, OS, + filesystem, standard virtual-environment layout, and installed + `.dist-info/RECORD` are trusted; +- the installed `RECORD` is mutable, unsigned, and may differ from the wheel + archive's original record after installer rewrites; coordinated + code-plus-`RECORD` changes are accepted; +- the file checks are individually race-aware but do not create a + transactionally atomic snapshot against a hostile concurrent mutator; +- the checks detect uncoordinated artifact drift; they do not prove publisher + identity, provenance, freshness, or an uncompromised host. + +Read [docs/artifact-boundary.md](docs/artifact-boundary.md) for the layer-by- +layer threat model and non-claims. + +The receipt does not embed the wheel digest, source commit, distribution +version, or evidence identity. Preserve the fresh-wheel verification manifest +beside any published receipt when artifact provenance matters. + +## Privacy protocol + +The parser is dependency-free and fail-closed: + +- strict UTF-8 and unambiguous IPv4/bracketed-IPv6 endpoint syntax; +- 8 MiB and 100,000-physical-line bounds; +- deterministic canonicalization and duplicate detection; +- explicit address-scope and IANA port-range counts; +- redacted parser failures that retain a bounded code and optional line number, + not the rejected endpoint. + +The report protocol uses separate versioned HMAC domains for source content +and repeated endpoint groups. Key rotation changes both typed identifiers. +Equivalent IPv6 spellings canonicalize before equality grouping. Canonical +JSON is byte-deterministic only for the same exact source bytes, key bytes, +code/artifact identity, and embedded Python runtime profile. + +[docs/privacy-protocol.md](docs/privacy-protocol.md) specifies the framing, +domains, schemas, ordering, runtime binding, count semantics, and disclosure +limits. + +## Library boundary + +Trusted local code can use the parser or report builder directly: + +```python +from netveil import build_privacy_receipt, parse_corpus + +payload = b"192.0.2.10:443\n[2001:db8::10]:8443\n" +corpus = parse_corpus(payload) +receipt = build_privacy_receipt( + payload, + pseudonymization_key=b"\x01" * 32, +) +``` + +This example key is deterministic test material, not production guidance. +Library calls—including an ordinary `import netveil` from the installed +wheel—do not use the installed-command guard. `Endpoint` and `EndpointCorpus` +models retain raw canonical addresses, and the corpus model retains an unkeyed +source SHA-256. Do not log or publish those objects when the input is +sensitive. + +## Developer checks + +Editable installation is for development only and is expected to fail the +guarded CLI contract. + +```bash +python3 -m venv .venv +.venv/bin/python -m pip install --requirement requirements-dev.txt +.venv/bin/python -m pip install --no-build-isolation --no-deps --editable . + +.venv/bin/ruff check . +.venv/bin/ruff format --check . +.venv/bin/mypy +.venv/bin/coverage run -m unittest discover -s tests +.venv/bin/coverage report -m +``` + +`requirements-dev.txt` pins the additional Python development packages used +by this repository on CPython 3.12/Linux. It is not a runtime dependency set +or a cross-platform, hash-locked software-supply-chain attestation. + +The current source gate covers the extensionless launcher, bootstrap, CLI, +parser, models, and privacy protocol at 100% statement and branch coverage. +A separate installed-artifact gate builds no trust from the editable +environment: it accepts an exact wheel, installs it into a fresh venv with +`--no-index --no-deps`, and exercises startup injection, bytecode, tamper, +determinism, and redaction cases. + +## Reproducible release evidence + +![Source-to-execution artifact provenance](docs/assets/artifact-provenance.svg) + +The publication gate requires Git, CPython 3.12, the pinned development +packages above, and Linux `strace`. Run it only from a clean checkout. The +builder enforces a clean Git top-level, exports the exact `HEAD` with +`git archive`, and rejects an output directory inside the source tree. + +```bash +set -eu + +COMMIT=$(git rev-parse --verify 'HEAD^{commit}') +SOURCE_DATE_EPOCH=$(git show -s --format=%ct "$COMMIT") +OUT="$(dirname "$PWD")/netveil-release-${COMMIT}" +test ! -e "$OUT" + +.venv/bin/python tools/build_release.py \ + "$PWD" "$OUT" \ + --source-date-epoch "$SOURCE_DATE_EPOCH" \ + --python "$PWD/.venv/bin/python" + +VERIFY_TMP="$OUT/.fresh-wheel-verification.json.tmp" +.venv/bin/python tools/verify_fresh_wheel.py \ + --source-commit "$COMMIT" \ + --inventory "$OUT/release-inventory.json" \ + --sdist "$OUT/netveil_audit-0.3.0.tar.gz" \ + "$OUT/netveil_audit-0.3.0-py3-none-any.whl" > "$VERIFY_TMP" +mv "$VERIFY_TMP" "$OUT/fresh-wheel-verification.json" +``` + +The verifier reads the actual wheel, sdist, and canonical inventory; installs +the pinned wheel into a fresh no-dependency environment; runs startup, +bytecode, tamper, deterministic-output, redaction, public-demo, and syscall +checks; and emits one path-free canonical JSON line. + +![Fresh-wheel verification matrix](docs/assets/verification-matrix.svg) + +The committed snapshot records exact source commit +`f3e4db737c4fd87b4b58f842a926ebb0f9889e69`: all 19 verifier checks passed, +both normalized Linux traces recorded zero network syscalls and zero +post-launch child processes, and the installed launcher and `RECORD` identities +were captured. Inspect the canonical +[release inventory](docs/evidence/release-inventory.json), the [fresh-wheel +verification record](docs/evidence/fresh-wheel-verification.json), and the +[visual manifest](docs/evidence/visual-manifest.json) rather than trusting the +rendered summary alone. + +`docs/evidence` is deliberately repository-only. Embedding an inventory that +hashes the sdist inside that same sdist would be self-referential; publish the +evidence beside the release artifacts, not inside them. + +To refresh the committed evidence views after a successful exact-commit run: + +```bash +mkdir -p docs/evidence +cp -- "$OUT/release-inventory.json" \ + docs/evidence/release-inventory.json +cp -- "$OUT/fresh-wheel-verification.json" \ + docs/evidence/fresh-wheel-verification.json + +.venv/bin/python tools/render_evidence.py +.venv/bin/python tools/render_evidence.py --check +``` + +Repeat the build into a second nonexistent sibling directory and compare the +wheel, sdist, and inventory bytes when testing build reproducibility. + +These JSON files are unsigned consistency and execution evidence, not a +signature or remote attestation. Their trusted computing base includes the +checked-out verifier, Git and `git archive`, CPython, `build`, setuptools, +pip/ensurepip, `strace`, the OS, and filesystem. They cannot prove that the +verifier or host was honest. Publish artifact digests through a separately +authenticated channel. The recorded trace hash covers normalized path-free +facts, not the temporary raw `strace` bytes. + +## Deliberate exclusions + +Netveil is not a scanner, proxy checker, service-discovery client, reachability +tester, anonymizer, key vault, package signature verifier, sandbox, or remote +attestation system. It never grants permission to test systems you do not own +or lack authorization to assess. + +The repository historically contained unverified third-party endpoint lists +collected in 2022. They were removed from the current tree because they had no +adequate provenance or consent. Their presence in Git history is not evidence +that any service is live and not authorization to connect to it. + +Use only caller-owned data, explicitly redistributable fixtures, or IETF +documentation ranges in examples and issues. See [SECURITY.md](SECURITY.md) +for disclosure and sensitive-data handling guidance. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..32dded3 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,153 @@ +# Security policy + +## Supported state + +Security work targets the default branch and the latest open rehabilitation +pull request. Netveil 0.3.0 is currently a release candidate, not a published +stable release. + +The supported guarded execution path is narrow: install an exact wheel into a +fresh, standard POSIX CPython virtual environment and invoke the installed +`bin/netveil-audit` script. Recorded evidence currently covers CPython 3.12.3 +on Linux x86-64. The library declares Python 3.11 or newer, but equivalent +launcher guarantees are not yet claimed for every interpreter or platform. + +Editable installs, direct source execution, ordinary `import netveil` library +use, `python -m netveil`, Windows, PyPy, Conda, `pip --target`, renamed or +externally symlinked launchers, and non-standard environment layouts are +outside the guarded 0.3.0 command contract. + +## Installed-command trust boundary + +The wheel contains a real POSIX shell/Python polyglot launcher. `/bin/sh` +executes the sibling virtual-environment interpreter with `-I -E -S -B` before +launcher Python code. The launcher checks that isolated, no-site, +environment-ignoring, no-bytecode-write profile; verifies its installed bytes +and `netveil_bootstrap.py` against selected entries in the installed +`.dist-info/RECORD`; and compiles the already-read bootstrap source in memory. + +The bootstrap checks a closed `netveil/` inventory, bounded file reads, +selected distribution metadata, and source hashes. It then installs a closed +finder that compiles pinned source bytes instead of importing source or +bytecode from disk. Preloaded and unknown `netveil.*` modules are rejected. + +These checks reduce startup injection and uncoordinated installation drift. +They are not self-authentication: + +- the launcher is already executing before its own later check; +- `/bin/sh`, sibling CPython, stdlib/import machinery, installer, OS, + filesystem, standard virtual-environment layout, and launcher are trusted; +- installed `.dist-info/RECORD` is mutable, unsigned consistency metadata, + can be rewritten by the installer, and is trusted; +- coherent changes to code and `RECORD` can pass; +- per-file identity rechecks do not form a transactionally atomic snapshot + against a hostile concurrent filesystem mutator; +- a privileged hostile host, kernel, debugger, ptrace client, injected native + code, malicious trusted stdlib, or memory/crash-dump collector is out of + scope. + +Verify a wheel SHA-256 through an authenticated channel before installation. +Netveil does not currently verify a package signature, publisher identity, +provenance attestation, or transparency-log entry. Full details and +unsupported paths are in +[docs/artifact-boundary.md](docs/artifact-boundary.md). + +## Local file handling + +After reading its installed launcher, metadata, and package files for +verification, the receipt workflow opens no caller data paths other than +`CORPUS` and `KEY_FILE`. Both must be direct POSIX regular files opened with +no-follow and nonblocking flags. `O_NOFOLLOW` applies to the final path +component; symlinks in parent directories are followed. The same descriptor's +identity is compared before and after each bounded read. + +The key must contain 32–4096 exact bytes, be owned by the effective user, have +one hard link, be owner-readable, and expose no owner-execute, group, other, or +special permission bits. The corpus cannot be the same inode as the key or a +byte-identical copy. Netveil does not measure key entropy or replace a secret +manager. Never put a production key in source control, logs, shell history, +report metadata, issue attachments, screenshots, or demo recordings. + +Corpus ownership, permission bits, and hard-link count are not restricted. +Input paths remain visible to the operating system and may be visible in a +process listing even though Netveil diagnostics do not repeat them. The caller +also controls stdout redirection, destination permissions, atomic publication, +and retention; Netveil writes receipt bytes to stdout rather than securely +creating an output file. + +The full syntax, size, permission, and exit-code contract is in +[docs/cli-contract.md](docs/cli-contract.md). + +## Privacy boundary + +`Endpoint` and `EndpointCorpus` retain raw canonical addresses. The corpus +model also retains an unkeyed source SHA-256. Treat those library objects, +caller input bytes, traceback frame locals, debuggers, and crash dumps as +sensitive. Redacted library exceptions protect the library-created message +and context; they cannot erase caller-owned objects or ambient exception +state. + +`build_privacy_report`, `build_privacy_receipt`, and the installed `receipt` +command keep raw endpoint strings, secret key bytes, and the ordinary source +digest out of their returned public models. Source-content and duplicate-group +identifiers use separate versioned HMAC-SHA256 domains and typed prefixes. + +Reports are pseudonymized, not anonymous. HMAC resists offline guessing only +while a high-entropy key remains secret. It does not stop an actor who can +submit chosen corpora to a report-generation oracle. Reusing a key deliberately +links exact source content and repeated endpoint groups across reports. +Published output also reveals source size, line count, category totals, +duplicate equality, and duplicate frequency. Review those disclosures before +publication. + +The report records `sys.implementation.name` and numeric Python +`major.minor.micro` whose stdlib `ipaddress` behavior supplied parsing and +classification. It does not hash the interpreter executable or stdlib. +Compare bytes only for the same trusted code artifact and runtime semantics. +Python cannot guarantee zeroization of source bytes, keys, canonical strings, +HMAC intermediates, or allocator copies. + +The receipt's SHA-256 binds canonical public report bytes. It is not a +signature, MAC, proof of origin, or authenticated publication channel. The +receipt does not identify the source commit, wheel digest, distribution +version, or fresh-wheel evidence manifest. The +byte-level protocol and remaining disclosure surface are specified in +[docs/privacy-protocol.md](docs/privacy-protocol.md). + +## Offline and authorization boundary + +Netveil package code contains no intentional DNS, socket, subprocess, or +child-process operation. A publishable installed-artifact evidence bundle must +pair hostile Python environment-variable tests with Linux syscall tracing for +one exact artifact and runtime. That trace is evidence for the recorded run, +not a guarantee about a compromised trusted interpreter, shell, operating +system, or preload mechanism. + +Release inventory and fresh-wheel verification JSON are unsigned. They bind a +builder-observed clean commit to exact artifact and executed-test facts, but +they do not authenticate the publisher, attest the build host, or prove that +Git, the verifier, build backend, installer, `strace`, OS, and filesystem were +honest. Obtain published artifact digests through a separately authenticated +channel. + +Netveil is not a scanner or reachability tester. Historical Git commits +contain unverified public endpoint strings removed from the current tree. +Their existence does not grant permission to connect to, probe, or test those +systems. Use only caller-owned data, explicitly redistributable fixtures, or +IETF documentation ranges. + +## Reporting a vulnerability + +Use GitHub private vulnerability reporting for this repository when available. +Include: + +- the affected commit and exact wheel SHA-256; +- interpreter, OS, install method, and invocation path; +- a minimal reproduction using synthetic data; +- the expected documented boundary and observed behavior; +- whether code and `RECORD` were changed independently or coherently. + +Do not attach real endpoint inventories, private-network layouts, credentials, +access tokens, keys, personal data, memory dumps, or absolute private paths. +If private reporting is unavailable, open a minimal issue that contains no +sensitive reproduction material and asks the maintainer for a secure channel. diff --git a/checker_2022-06-04_10.33.29.txt b/checker_2022-06-04_10.33.29.txt deleted file mode 100644 index 35e7bfe..0000000 --- a/checker_2022-06-04_10.33.29.txt +++ /dev/null @@ -1,15 +0,0 @@ -62.113.115.94:16072 -72.195.114.169:4145 -72.195.34.58:4145 -72.49.49.11:31034 -5.161.93.53:1080 -192.252.214.20:15864 -208.102.51.6:58208 -70.166.167.55:57745 -72.221.196.157:35904 -67.201.33.10:25283 -72.221.172.203:4145 -218.64.255.198:7302 -193.252.50.1:3820 -67.201.33.9:25280 -150.136.189.225:80 \ No newline at end of file diff --git a/docs/artifact-boundary.md b/docs/artifact-boundary.md new file mode 100644 index 0000000..3b18e6e --- /dev/null +++ b/docs/artifact-boundary.md @@ -0,0 +1,175 @@ +# Installed artifact boundary + +Netveil's installed guard reduces accidental drift and Python import-path +ambiguity. It is not package signing, sandboxing, or host attestation. + +## Execution sequence + +The supported wheel installs a real POSIX shell/Python polyglot script rather +than a generated `console_scripts` wrapper: + +```text +wheel-installed netveil-audit + -> /bin/sh + -> sibling venv/bin/python -I -E -S -B + -> static launcher checks startup and installed-RECORD-bound files + -> installed-RECORD-bound netveil_bootstrap.py is compiled from pinned bytes + -> package inventory and source records are checked + -> netveil.* is compiled from pinned source bytes by a closed finder + -> receipt command reads local corpus/key files + -> canonical pseudonymized JSON +``` + +The combined Python flags take effect before launcher Python code: + +- `-I`: isolated mode and safe import path; +- `-E`: ignore Python environment variables; +- `-S`: do not import `site` or `sitecustomize`; +- `-B`: do not write bytecode caches. + +The launcher verifies those flags at runtime. It locates one exact standard +virtual-environment site directory, preflights installed `METADATA` and +`RECORD` as regular files of at most 1,048,576 bytes, checks the installed +script and `netveil_bootstrap.py` against their SHA-256/size entries in the +installed `.dist-info/RECORD`, and compiles the already-read bootstrap bytes +in memory. It never imports the top-level bootstrap by name. + +`importlib.metadata` subsequently parses those trusted installed metadata +files and may reread them. The preflight is a resource bound, not an +authentication step or atomic snapshot. Selected metadata payloads and every +record-bound launcher/bootstrap/package file have the same 1,048,576-byte +per-file runtime ceiling. + +The bootstrap then: + +- requires the exact distribution name and version; +- requires no entry points and the exact `top_level.txt` bytes + `netveil\nnetveil_bootstrap\n`; +- checks selected metadata and every allowed `netveil/` source record; +- opens files with bounded, no-follow descriptor reads and identity rechecks; +- rejects unknown files inside the package directory; +- treats allowed package bytecode as inert and rejects unknown package cache + names; +- refuses a preloaded `netveil` package; +- installs a closed first-position finder; +- compiles only the pinned source bytes already held in memory; +- rejects imports of unlisted `netveil.*` modules. + +## Trust table + +| Layer | Status | +|---|---| +| authenticated wheel SHA-256 obtained before install | external trust root | +| installer and install-time environment | trust root | +| operating system and filesystem semantics | trust root | +| `/bin/sh` and sibling CPython interpreter | trust root | +| CPython stdlib and import machinery | trust root | +| installed static launcher | directly executed trust root | +| standard POSIX virtual-environment layout | required precondition | +| installed `.dist-info/RECORD` | installer-derived mutable consistency input and trust root | +| `netveil_bootstrap.py` bytes | checked against installed `RECORD` before execution | +| `netveil.*` source bytes | checked, pinned, and compiled in memory | +| corpus and key files | sensitive caller-controlled inputs | +| canonical receipt | public pseudonymized output, subject to disclosure review | + +## Non-claims + +The runtime never reads the wheel archive or verifies its SHA-256. It reads +the installer-produced `.dist-info/RECORD`, whose launcher path and bytes can +differ from the wheel archive's original record after installation. +`RECORD` is unsigned. A party able to change code and update it coherently can +make the consistency checks accept those new bytes. The guard therefore does +**not** prove authenticity, publisher identity, provenance, freshness, or +installation from a particular wheel. Verify a published wheel digest through +an authenticated channel before installing it. + +The launcher is already executing when it checks its own current bytes. A +modified launcher can run code before its later mismatch is detected. The +launcher, `/bin/sh`, interpreter, stdlib, OS, filesystem, and `RECORD` remain +inside the trusted computing base. + +The runtime does not hash the sibling Python interpreter, `/bin/sh`, installer, +or stdlib. It does not enforce owner or write-mode policy for installed +artifact files; the launcher checks only that the installed script is a +regular file. The separate release-evidence gate additionally requires the +wheel and installed launcher modes to be owner-executable. A compromised +installer or install-time host can coherently alter installed files and +`RECORD`. + +`-B` prevents bytecode writes; it is not the reason existing bytecode is inert. +The top-level bootstrap is compiled directly from checked source bytes, and +package modules are served by the closed in-memory source finder. Unknown +package cache files are rejected. + +The guard is not a sandbox and does not defend against a privileged hostile +host, kernel compromise, debugger, ptrace, process-memory reader, injected +native code, malicious trusted stdlib, crash-dump collector, or coherently +mutated trust roots. Per-file identity checks do not create a transactionally +atomic snapshot across the whole installation against a hostile concurrent +filesystem mutator. Python cannot guarantee secret-memory zeroization. + +The runtime bootstrap verifies selected distribution records and the closed +`netveil/` package directory. It does not enforce the exact full wheel or sdist +inventory; that is a separate release gate. + +The receipt schema itself contains no wheel digest, source commit, +distribution version, or evidence-manifest identifier. Those provenance +bindings remain external release evidence. + +## Unsupported invocation paths + +The following are deliberately outside the 0.3.0 guarded command contract: + +- editable installs and direct source-checkout execution; +- ordinary `import netveil` library use, including from the installed wheel; +- `python -m netveil`; +- invoking the launcher with ordinary Python flags instead of its shell + handoff; +- Windows, PyPy, Conda, `pip --target`, and non-standard install layouts; +- renamed launcher files or invocation through external symlinks; +- preloaded or embedded `netveil` module state. + +A renamed or expected-name symlink is rejected once Python starts. An external +symlink without the expected adjacent interpreter can fail in the shell before +Netveil controls diagnostics; shell output may then include the invocation +path. Use the installed script in the environment's own `bin/` directory. + +## Evidence standard + +A publishable artifact must be checked from a clean source commit and fresh +wheel installation. The gate must bind: + +- source commit, wheel SHA-256, interpreter version, and exact archive + inventory; +- launcher bytes, mode, shebang, installed `RECORD`, and absence of + `entry_points.txt`; +- isolated startup under hostile Python environment variables; +- inert bootstrap/package bytecode and rejection of unknown package files; +- fail-closed uncoordinated source and metadata tampering; +- deterministic receipt bytes from synthetic IETF documentation ranges; +- absence of raw endpoint and key material in public outputs; +- an offline syscall trace for the exact tested Linux execution. + +Coordinated code-plus-`RECORD` mutations belong in negative trust-boundary +evidence: they must be shown as accepted, not misleadingly presented as +attacks the unsigned guard prevents. + +## Release-evidence trust boundary + +The release builder and verifier establish an internally consistent chain: +clean Git `HEAD` to `git archive`, wheel and canonical sdist bytes, fresh +installation, selected installed files, executed checks, and normalized +process/network trace facts. The release inventory also carries the exact +safe sdist member inventory. + +That chain is unsigned. It trusts the checked-out builder, verifier, and +renderer sources; Git and `git archive`; CPython, `build`, and setuptools; +pip and ensurepip; `strace`; the operating system and filesystem; and the +channel used to publish the resulting digests. It does not prove that those +components were unmodified, that the build or CI host was honest, or that the +person publishing the manifest is Omar. Publisher authentication remains an +external release-channel responsibility. + +The verifier publishes a digest of normalized, path-free trace facts together +with the exact exec counts and zero network/child-process counts. It does not +publish or claim a digest of the temporary raw `strace` files. diff --git a/docs/assets/architecture.svg b/docs/assets/architecture.svg new file mode 100644 index 0000000..4688844 --- /dev/null +++ b/docs/assets/architecture.svg @@ -0,0 +1,56 @@ + +Netveil installed execution and trust boundary +Code-derived architecture from the installed polyglot launcher through pinned source loading to a pseudonymized receipt. + + + + + +Netveil installed execution boundary +Generated from the 0.3.0 launcher, parser, CLI and bootstrap sources + +1 Trusted startup +/bin/sh → sibling CPython +combined flags: -I -E -S -B +no site hooks • env ignored • safe path +launcher + installed RECORD are trust roots + +2 Artifact drift guard +launcher and bootstrap: installed hash + size +bounded O_NOFOLLOW descriptor reads +identity checked before and after read +installed RECORD is not a signature + +3 Closed source loader +5 allowed modules +pinned source bytes compiled in memory +unknown netveil.* imports rejected +package bytecode never executes + + + +4 Sensitive local inputs +corpus: regular file • ≤ 8 MiB • ≤ 100,000 lines +key: owner-only regular file • 32–4,096 exact bytes +same inode or byte-identical corpus/key rejected +sensitive values omitted from handled output +no DNS, socket or child-process workflow + +5 Public deterministic result +parse → canonicalize → aggregate +domain-separated HMAC-SHA256 identifiers +fixed-schema sorted-key canonical JSON +receipt digest binds the public report +pseudonymized, not anonymous + + + +Exact source inventory +netveil, netveil.cli, netveil.model, netveil.parser, netveil.privacy +Trust-root legend +amber = executes before or supplies the mutable consistency boundary +Verified-byte legend +cyan / blue = checked before compilation by the supported installed path +Reproduce: python3 tools/render_architecture.py --check +No network data or endpoint fixture is used by this diagram + diff --git a/docs/assets/artifact-provenance.svg b/docs/assets/artifact-provenance.svg new file mode 100644 index 0000000..ea93381 --- /dev/null +++ b/docs/assets/artifact-provenance.svg @@ -0,0 +1,59 @@ + +Netveil release evidence chain +Source commit, artifact inventory, installed-file, and fresh execution evidence with an explicit unsigned non-claim. + + +Release evidence chain +Every displayed value comes from the canonical release inventory or fresh-wheel verifier JSON. + + +1 · clean Git HEAD +f3e4db737c4fd87b4b58f842a926ebb0f9889e69 +SOURCE_DATE_EPOCH 1785338401 +BUILDER-OBSERVED + + + + +2 · bound release artifacts +wheel · 25.8 KiB · 0f4254362623a344e247… +sdist · 105.3 KiB · 51 members +BOUND IN EVIDENCE + + + + +3 · fresh installed state +launcher 0711 · 8.5 KiB +RECORD 0600 · 13 selected rows +BOUND IN EVIDENCE + + + + +4 · executed evidence +19 checks pass +2 traces · network 0 · children 0 +BOUND IN EVIDENCE + +Exact installed bindings +launcher sha256 056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac +RECORD sha256 88c0fe5e9468c9b99aeac359f8a7b71df99b546e4ba73cdce8561cfe5fc002e0 +inventory sha256 e41685491e63cdfb8860ec7cce48ca1f8743da6d29cc7b5d3c398f738c1c6622 + +Claim boundary +The manifest is unsigned. It shows internally consistent bytes and an executed test run; it does not prove publisher identity, an honest host, or an unmodified verifier. +Authenticate the published artifact digests through a separate trusted channel. + diff --git a/docs/assets/cli-session.svg b/docs/assets/cli-session.svg new file mode 100644 index 0000000..7df6811 --- /dev/null +++ b/docs/assets/cli-session.svg @@ -0,0 +1,43 @@ + +Netveil installed CLI evidence +Terminal rendering of exact version and receipt standard output captured from the verified fresh wheel. + + +Installed CLI · captured stdout +Exact successful output captured by the fresh-wheel verifier; long canonical JSON is visually wrapped only. + + + + +Netveil installed-wheel public demonstration +source commit: f3e4db737c4fd87b4b58f842a926ebb0f9889e69 +input class: synthetic IETF documentation ranges +key class: public non-secret test material + +$ netveil-audit --version +netveil-audit 0.3.0 +$ netveil-audit receipt documentation-corpus.txt --key-file public-demo.key +{"canonicalization":"netveil.sorted-keys-json.v1","protocol":"netveil.hmac-sha256-pseudonymization.v1","report":{"canonicaliza + tion":"netveil.sorted-keys-json.v1","counts":{"endpoint_occurrences":5,"physical_lines":6,"source_bytes":129,"unique_endpoin + ts":4},"duplicates":{"extra_occurrences":1,"group_count":1,"groups":[{"extra_occurrences":1,"id":"nvd1_f382d197a21771fcdbc8d + a8d818d4612a6a6bf7d8ca45415298f475b5709da10","occurrences":2}]},"endpoint_occurrences_by_ip_version":{"ipv4":4,"ipv6":1},"en + dpoint_occurrences_by_port_bucket":{"dynamic_49152_65535":1,"registered_1024_49151":1,"system_1_1023":3},"endpoint_occurrenc + es_by_scope":{"documentation":5,"global":0,"link_local":0,"loopback":0,"multicast":0,"private":0,"reserved":0,"shared":0,"si + te_local":0,"unspecified":0},"protocol":"netveil.hmac-sha256-pseudonymization.v1","runtime":{"endpoint_semantics":"python-st + dlib-ipaddress","python_implementation":"cpython","python_version":"3.12.3","schema":"netveil.python-runtime.v1"},"schema":" + netveil.aggregate-report.v1","source_content_id":"nvs1_db2e7cafc475293724cdaa15fd267fd750a46f33ea199091a14cfa7635523e31"},"r + eport_digest":{"algorithm":"sha256","value":"e266ff9c6b81df76e9224772d67958be75b90b9a8ac67161182cf39985b76f1e"},"schema":"ne + tveil.aggregate-receipt.v1"} + diff --git a/docs/assets/receipt-counts.svg b/docs/assets/receipt-counts.svg new file mode 100644 index 0000000..659c670 --- /dev/null +++ b/docs/assets/receipt-counts.svg @@ -0,0 +1,53 @@ + +Netveil public receipt count evidence +Bar chart of physical lines, endpoint counts, IP versions, and duplicate groups parsed from the committed public demo receipt. + + +Public receipt · disclosed counts +Values parsed from the exact verifier-captured receipt; identifiers and raw endpoints are not used in this chart. + +physical lines + + +6 +endpoint occurrences + + +5 +unique endpoints + + +4 +IPv4 occurrences + + +4 +IPv6 occurrences + + +1 +duplicate groups + + +1 + +Scope result +5 / 5 occurrences · documentation ranges +Synthetic corpus only; this is workflow evidence, not a claim about live infrastructure. + +Disclosure boundary +129 source bytes are summarized, not embedded +Counts, equality and frequency remain public; pseudonymized does not mean anonymous. + diff --git a/docs/assets/verification-matrix.svg b/docs/assets/verification-matrix.svg new file mode 100644 index 0000000..3ecfdcf --- /dev/null +++ b/docs/assets/verification-matrix.svg @@ -0,0 +1,110 @@ + +Netveil fresh-wheel verification matrix +All verifier pass labels, interpreter and platform identity, and normalized path-free process and network trace evidence. + + +Fresh-wheel verification matrix +19 exact checks reported pass by the committed verifier JSON. + +cpython 3.12.3 · cpython-312 +Linux 6.17.0-1019-aws · x86_64 +f3e4db737c4fd87b4b58f842a926ebb0f9889e69 +unsigned consistency evidence · not attestation + + +wheel archive +PASS + + +fresh install +PASS + + +entry points absent +PASS + + +launcher identity +PASS + + +direct and path commands +PASS + + +isolated startup +PASS + + +environment injection inert +PASS + + +unchecked bytecode inert +PASS + + +unknown bytecode rejected +PASS + + +tamper fail closed +PASS + + +metadata drift rejected +PASS + + +coordinated bootstrap record mutation accepted +PASS + + +unknown package file rejected +PASS + + +receipt deterministic +PASS + + +receipt redacted +PASS + + +public demo capture +PASS + + +syscall trace offline +PASS + + +release inventory integrity +PASS + + +source commit bound +PASS +Normalized offline syscall evidence + +receipt +exec=2 · network=0 · post-launch children=0 +normalized sha256 0cdfc79931d31b9665f522437c58fb2df4b7a30cfdf69fccec996a206b35721a + +version +exec=2 · network=0 · post-launch children=0 +normalized sha256 f49c6be35c867adf767a07fcd01788478f57ecfddee88550f992f9275d3d7b57 + diff --git a/docs/cli-contract.md b/docs/cli-contract.md new file mode 100644 index 0000000..8ca314a --- /dev/null +++ b/docs/cli-contract.md @@ -0,0 +1,141 @@ +# Installed CLI contract + +This document specifies Netveil 0.3.0's local, wheel-installed command. It is +an interface contract, not a claim that mutable Python packaging metadata is a +signature. The exact artifact and trust boundary are documented in +[artifact-boundary.md](artifact-boundary.md). + +## Supported command surface + +The command grammar is deliberately small: + +```text +netveil-audit --help +netveil-audit --version +netveil-audit receipt CORPUS --key-file KEY_FILE +``` + +`--version` is accepted only as the sole argument; mixing it with a receipt, +another option, or a second `--version` is a usage error. Top-level and +`receipt` help remain available through argparse's ordinary `-h`/`--help` +forms. Long options are not abbreviated, and `--key-file` must appear exactly +once in a receipt invocation. + +The supported guarded path is the `netveil-audit` script installed from a +wheel into the standard `bin/` directory of a POSIX CPython virtual +environment. Netveil 0.3.0's recorded execution evidence uses CPython 3.12.3 +on Linux x86-64. The package declares Python 3.11 or newer, but the guarded +launcher is not yet claimed as verified on every such interpreter or on +Windows, PyPy, Conda, editable installs, `pip --target`, or non-standard +environment layouts. + +`python -m netveil` is intentionally unavailable. Running the source checkout, +the launcher through an ordinary non-isolated Python command, or an editable +install is not a substitute for the installed guard. + +## Corpus file + +`CORPUS` must resolve directly to one POSIX regular file. The command opens it +read-only with `O_CLOEXEC`, `O_NOCTTY`, `O_NOFOLLOW`, and `O_NONBLOCK`, then +checks the same descriptor's identity before and after a bounded read. +`O_NOFOLLOW` applies only to the final path component; symlinks in parent +directories are followed. Corpus owner, permission bits, and hard-link count +are not restricted. + +The parser accepts at most 8 MiB and 100,000 physical lines. Input must be +strict UTF-8. Empty lines and lines beginning with `#` are ignored; other lines +must contain an IPv4 literal in `IPv4:port` form or an IPv6 literal in +bracketed `[IPv6]:port` form. Accepted addresses are canonicalized before +grouping. Ports are decimal integers from 1 through 65535 with no leading +zeroes. CRLF and LF are accepted; ambiguous or non-LF Unicode separators are +rejected. + +No address is resolved or contacted. The file can contain private data, so its +path and rejected endpoint text are never repeated in Python-handled Netveil +diagnostics. The input paths remain process arguments and can be visible to the +operating system or process observers. + +## Key file + +`KEY_FILE` is read as exact bytes. It is not decoded, trimmed, or normalized; +a trailing newline is therefore part of the key. The file must satisfy every +condition below: + +- one POSIX regular file, not a symlink, FIFO, socket, or directory; +- 32 through 4096 bytes inclusive; +- owned by the process's effective user ID; +- exactly one hard link; +- readable by the owner; +- no owner execute bit; +- no group or other permission bits; +- no set-user-ID, set-group-ID, or sticky bit. + +Mode `0400` or `0600` is accepted. The corpus and key must not be the same +inode and must not contain byte-identical payloads. Those checks prevent a +corpus from being accidentally reused as its own pseudonymization key; they do +not assess entropy or operate a key-management system. + +## Output + +On success, `receipt` writes one canonical +`netveil.aggregate-receipt.v1` JSON document followed by exactly one LF to +standard output. Standard error is empty. + +For the same exact code/artifact identity, corpus bytes, key bytes, and +embedded Python runtime profile, the output bytes are deterministic. Different +Python micro versions are not promised to be byte-identical because the +runtime profile and stdlib `ipaddress` semantics are inside the receipt's +digest boundary. + +The receipt contains pseudonymized aggregate data, not anonymous data. It +reveals input size, physical-line count, category totals, duplicate equality, +and duplicate frequency. It contains neither raw endpoint strings, key bytes, +nor an unkeyed digest of the corpus. See +[privacy-protocol.md](privacy-protocol.md) for the byte-level schema and HMAC +domains. + +Netveil writes the receipt only to stdout. Shell redirection, downstream pipe +behavior, destination permissions, atomic publication, and retention are the +caller's responsibility. The receipt does not embed the wheel SHA-256, source +commit, distribution version, or fresh-wheel evidence identity. + +## Exit codes + +Diagnostics are stable, one-line, and intentionally omit caller paths and +input values. + +| Exit | Diagnostic or result | +|---:|---| +| 0 | receipt, help, or version completed | +| 2 | `netveil-audit: usage_error` | +| 10 | `netveil-audit: artifact_unverified` or `netveil-audit: platform_unsupported` from the bootstrap | +| 11 | `netveil-audit: corpus_unavailable` | +| 12 | `netveil-audit: key_unavailable` | +| 13 | `netveil-audit: corpus_rejected:[:line=N]` | +| 14 | `netveil-audit: key_rejected` | +| 15 | `netveil-audit: output_failed` | +| 70 | `netveil-audit: internal_error` or `netveil-audit: interrupted` | + +If standard error itself cannot make bounded forward progress, the process +returns 15 and may be unable to emit the complete diagnostic. + +Oversized, non-regular, symlinked, unavailable, or identity-changing corpus +files fail at the file boundary with exit 11. The equivalent key-file cases +use exit 12. A short or policy-insecure key, or a key that is the same inode or +byte-identical payload as the corpus, uses exit 14. Because the command's +8 MiB file read bound runs before parsing, an oversized installed-CLI corpus +does not reach the parser's library-level `input_too_large` code. + +Treat stdout as valid only when the process exits 0. An output failure can +leave a partial stdout prefix before exit 15. Failures in `/bin/sh` or sibling +interpreter execution occur before Netveil's Python error boundary and can use +OS-defined exit codes and shell diagnostics, including an invocation path. + +## Offline scope + +Netveil package code exposes no DNS, socket, subprocess, or child-process path. +A publishable release-evidence bundle must exercise the exact installed +artifact under syscall tracing and hostile Python environment variables. Such +evidence describes the tested Linux run; it is not a mathematical guarantee +about a compromised interpreter, shell, operating system, debugger, or preload +mechanism outside the stated trust boundary. diff --git a/docs/evidence/cli-session.cast b/docs/evidence/cli-session.cast new file mode 100644 index 0000000..b633f91 --- /dev/null +++ b/docs/evidence/cli-session.cast @@ -0,0 +1,6 @@ +{"env":{"SHELL":"/bin/sh","TERM":"xterm-256color"},"height":40,"timestamp":1785338401,"version":2,"width":120} +[0.1,"o","Netveil verified public demo\r\nsynthetic IETF documentation ranges \u00b7 public non-secret key\r\n\r\n"] +[0.6,"o","$ netveil-audit --version\r\n"] +[0.9,"o","netveil-audit 0.3.0\r\n"] +[1.4,"o","$ netveil-audit receipt documentation-corpus.txt --key-file public-demo.key\r\n"] +[1.8,"o","{\"canonicalization\":\"netveil.sorted-keys-json.v1\",\"protocol\":\"netveil.hmac-sha256-pseudonymization.v1\",\"report\":{\"canonicalization\":\"netveil.sorted-keys-json.v1\",\"counts\":{\"endpoint_occurrences\":5,\"physical_lines\":6,\"source_bytes\":129,\"unique_endpoints\":4},\"duplicates\":{\"extra_occurrences\":1,\"group_count\":1,\"groups\":[{\"extra_occurrences\":1,\"id\":\"nvd1_f382d197a21771fcdbc8da8d818d4612a6a6bf7d8ca45415298f475b5709da10\",\"occurrences\":2}]},\"endpoint_occurrences_by_ip_version\":{\"ipv4\":4,\"ipv6\":1},\"endpoint_occurrences_by_port_bucket\":{\"dynamic_49152_65535\":1,\"registered_1024_49151\":1,\"system_1_1023\":3},\"endpoint_occurrences_by_scope\":{\"documentation\":5,\"global\":0,\"link_local\":0,\"loopback\":0,\"multicast\":0,\"private\":0,\"reserved\":0,\"shared\":0,\"site_local\":0,\"unspecified\":0},\"protocol\":\"netveil.hmac-sha256-pseudonymization.v1\",\"runtime\":{\"endpoint_semantics\":\"python-stdlib-ipaddress\",\"python_implementation\":\"cpython\",\"python_version\":\"3.12.3\",\"schema\":\"netveil.python-runtime.v1\"},\"schema\":\"netveil.aggregate-report.v1\",\"source_content_id\":\"nvs1_db2e7cafc475293724cdaa15fd267fd750a46f33ea199091a14cfa7635523e31\"},\"report_digest\":{\"algorithm\":\"sha256\",\"value\":\"e266ff9c6b81df76e9224772d67958be75b90b9a8ac67161182cf39985b76f1e\"},\"schema\":\"netveil.aggregate-receipt.v1\"}\r\n"] diff --git a/docs/evidence/fresh-wheel-verification.json b/docs/evidence/fresh-wheel-verification.json new file mode 100644 index 0000000..4a18f6a --- /dev/null +++ b/docs/evidence/fresh-wheel-verification.json @@ -0,0 +1 @@ +{"checks":[{"name":"wheel_archive","status":"pass"},{"name":"fresh_install","status":"pass"},{"name":"entry_points_absent","status":"pass"},{"name":"launcher_identity","status":"pass"},{"name":"direct_and_path_commands","status":"pass"},{"name":"isolated_startup","status":"pass"},{"name":"environment_injection_inert","status":"pass"},{"name":"unchecked_bytecode_inert","status":"pass"},{"name":"unknown_bytecode_rejected","status":"pass"},{"name":"tamper_fail_closed","status":"pass"},{"name":"metadata_drift_rejected","status":"pass"},{"name":"coordinated_bootstrap_record_mutation_accepted","status":"pass"},{"name":"unknown_package_file_rejected","status":"pass"},{"name":"receipt_deterministic","status":"pass"},{"name":"receipt_redacted","status":"pass"},{"name":"public_demo_capture","status":"pass"},{"name":"syscall_trace_offline","status":"pass"},{"name":"release_inventory_integrity","status":"pass"},{"name":"source_commit_bound","status":"pass"}],"installed":{"launcher":{"logical_path":"bin/netveil-audit","mode":"0711","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},"record":{"logical_path":"site-packages/netveil_audit-0.3.0.dist-info/RECORD","mode":"0600","sha256":"88c0fe5e9468c9b99aeac359f8a7b71df99b546e4ba73cdce8561cfe5fc002e0","size_bytes":1609},"selected_record_rows":[{"path":"../../../bin/netveil-audit","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},{"path":"netveil/__init__.py","sha256":"ce9ae0bfd8d5080d58e86c661ddfe910d86894cd99ef7aca58053c4fdc5cfb6e","size_bytes":1216},{"path":"netveil/cli.py","sha256":"577c91cf2a11aae46273e3ebe23ecab1f6e9834261de5b914d1644ac1552e243","size_bytes":11078},{"path":"netveil/model.py","sha256":"1f333d4152ad8b956c971595300b49984a56dc9b9dd90e8cd32157f3fdffc4f3","size_bytes":6026},{"path":"netveil/parser.py","sha256":"ee9c20174454cf8029432543d4358a71d77da0f837b13a3b375ef9d2d7179e8a","size_bytes":5106},{"path":"netveil/privacy.py","sha256":"fba03e6d5a7d618797a0654252394af04f88069ddc5da13ada34199d62b5cbfd","size_bytes":18224},{"path":"netveil/py.typed","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"path":"netveil_audit-0.3.0.dist-info/METADATA","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"path":"netveil_audit-0.3.0.dist-info/RECORD","sha256":null,"size_bytes":null},{"path":"netveil_audit-0.3.0.dist-info/WHEEL","sha256":"2b6eb4118ce7cd7b09601406aa623c553c4476265836f0d9c16f5c061f7efcc0","size_bytes":91},{"path":"netveil_audit-0.3.0.dist-info/licenses/LICENSE","sha256":"125e5c962ef12e9a92e39f9af7776d8010a98fce5f4559221be7033573c226a8","size_bytes":1069},{"path":"netveil_audit-0.3.0.dist-info/top_level.txt","sha256":"9f7e05afb0b8dbbd49796a869258dc8cdc13dae0973dd848872799554d018ec7","size_bytes":26},{"path":"netveil_bootstrap.py","sha256":"71205f05f3398b7ddc6ccf4b820c001f71f2932694f370dfccee2d31a33f9de8","size_bytes":16752}]},"integrity_evidence":{"artifacts":[{"filename":"netveil_audit-0.3.0-py3-none-any.whl","kind":"wheel","sha256":"0f4254362623a344e2474db96be81534f502e291b990fcf9ab1f0ba46d0f0675","size_bytes":26440},{"filename":"netveil_audit-0.3.0.tar.gz","kind":"sdist","members":[{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/LICENSE","sha256":"125e5c962ef12e9a92e39f9af7776d8010a98fce5f4559221be7033573c226a8","size_bytes":1069},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/MANIFEST.in","sha256":"8fab5c0ebe83e4fe3ce4118045e2cbe48c70b4b60aabaadb9d06d335081f01d3","size_bytes":430},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/PKG-INFO","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/README.md","sha256":"97eb7a6405c8f2360d147fab133fab614e5fa661086a0b75bc3df2a7f62401b6","size_bytes":10824},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/SECURITY.md","sha256":"2e4f7ddfaf851ca804f2d9b6306cdda309646aa0dd87ed1f33ffec9745121201","size_bytes":8028},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/docs","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/artifact-boundary.md","sha256":"91b2026791ce76c3ab1662b79ea59649f9d6d31ab07aca7e0b4f80ba2c0361d5","size_bytes":8545},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/docs/assets","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/assets/architecture.svg","sha256":"5adc28a9088dc6f000ce810511726fe6d14cfcb6996fbff76a6e3f9154f28abc","size_bytes":8558},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/cli-contract.md","sha256":"6a1b32fed2e7fee8d0c44a1b2a1f70cb0153b63488ec9e7ca04371160484a59d","size_bytes":6541},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/privacy-protocol.md","sha256":"d047daf39cb49b69ff9c486aa0a7e9e06e9d872eb8cadfa5e13eb3bd9e2804f1","size_bytes":5213},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/pyproject.toml","sha256":"af21a11f116f31b5f0783934e1583ac1984f4c8f7dea0512812168ae47248299","size_bytes":1445},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/requirements-dev.txt","sha256":"6ca7f1739fb5f044210252ca5e6893bc1fee5ceb30130aaa44731738d253cef4","size_bytes":348},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/scripts","size_bytes":0},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/scripts/netveil-audit","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/setup.cfg","sha256":"1c473cbaee8da5fc46e7f0158794af5cea4414c34a3cf3f180c2001f5e38bd3e","size_bytes":38},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src","size_bytes":0},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src/netveil","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/__init__.py","sha256":"ce9ae0bfd8d5080d58e86c661ddfe910d86894cd99ef7aca58053c4fdc5cfb6e","size_bytes":1216},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/cli.py","sha256":"577c91cf2a11aae46273e3ebe23ecab1f6e9834261de5b914d1644ac1552e243","size_bytes":11078},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/model.py","sha256":"1f333d4152ad8b956c971595300b49984a56dc9b9dd90e8cd32157f3fdffc4f3","size_bytes":6026},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/parser.py","sha256":"ee9c20174454cf8029432543d4358a71d77da0f837b13a3b375ef9d2d7179e8a","size_bytes":5106},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/privacy.py","sha256":"fba03e6d5a7d618797a0654252394af04f88069ddc5da13ada34199d62b5cbfd","size_bytes":18224},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/py.typed","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/PKG-INFO","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/SOURCES.txt","sha256":"97e233d844643aa7fa463361c581cf46e4a6c6d0cc296c51758f8431d30ac231","size_bytes":958},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/dependency_links.txt","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/top_level.txt","sha256":"9f7e05afb0b8dbbd49796a869258dc8cdc13dae0973dd848872799554d018ec7","size_bytes":26},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_bootstrap.py","sha256":"71205f05f3398b7ddc6ccf4b820c001f71f2932694f370dfccee2d31a33f9de8","size_bytes":16752},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tests","size_bytes":0},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tests/fixtures","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/fixtures/launcher_bootstrap_fixture.py","sha256":"ce44795d98e969bd625373e9e84064c013470d641e009989e569e68d2ef95b73","size_bytes":251},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_bootstrap.py","sha256":"9b9f18cff5cc8a3499a6617861e4e8e7817ecba8495637a9f663afc95e971ea0","size_bytes":40030},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_cli.py","sha256":"8f8ea9b0617262ef74dcd148f5933f39d74b067be4b54eefff3f438b4b68308e","size_bytes":27081},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_evidence_renderer.py","sha256":"76b4f6d9b4b9c350a1fcbdb32e7985ff64a2fc40c3c1f15622615235656b9c8c","size_bytes":11137},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_fresh_wheel_verifier.py","sha256":"0ed6c2d4f8c8ee65c5731005c7481b7c44655ae3fc8d9063fc455152e1f092ff","size_bytes":32914},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_launcher.py","sha256":"f9931cb6ec9e43ffc55ea9e89ee2188a3edede24df351ecd3c793a649a1f202e","size_bytes":21478},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_model.py","sha256":"fb54ea5687906b0410c0623626b2a16bc6b2b634bef17bb402811986b921b387","size_bytes":4737},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_packaging.py","sha256":"75eb1770f14b15c2baa4d8e1c3551c122d0de365a797d3849f4690e82de719d5","size_bytes":2603},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_parser.py","sha256":"bc606c4b0a7cb2c5fce74cba4133f9c625ea412c0d42670605e14e755d7305ab","size_bytes":10625},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_privacy.py","sha256":"d770c688d2dd13d0078c679dce4e2095c7a9c970a8e3c9a1d1194a7a779ad4f1","size_bytes":23160},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_release_builder.py","sha256":"b93c280b25f056c71d3c8038decc91185310d28f4af7bfb8a429b1f7c915a38e","size_bytes":20067},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_visuals.py","sha256":"5ec195c534d085b4ca59a578a54e177255da35d01a6deac419325dd786023df5","size_bytes":721},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tools","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tools/__init__.py","sha256":"28db1b0a84cbcf8a228da02aae487a38895d51b66e2ea37067058252351b23db","size_bytes":55},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/build_release.py","sha256":"979ca44ad2121588c4ddf72d721f4d5d49449546a8f5a1f9e06796d39d9e820d","size_bytes":25873},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/render_architecture.py","sha256":"25f308aa838397bf3e331351ca6781267b49b43069e066d78ad4a3acf34ae741","size_bytes":9738},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/render_evidence.py","sha256":"9b00b1a58e61d92b899d86d9777f720a8565aafdfa42b25d34e2aa505d068172","size_bytes":42639},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/verify_fresh_wheel.py","sha256":"c88109e00821895446ff4ff71bef8a283af0c70fa723e9e2a7ed630ea369f6f6","size_bytes":79572}],"sha256":"7d1975194f8a439539f10e50f71b4f37da1dc335569d941e4f25990d4cb14139","size_bytes":107816}],"attestation_verified":false,"inventory_schema":"netveil.release-inventory.v1","inventory_sha256":"e41685491e63cdfb8860ec7cce48ca1f8743da6d29cc7b5d3c398f738c1c6622","inventory_type":"unsigned_sha256_manifest","signature_verified":false,"source_commit":"f3e4db737c4fd87b4b58f842a926ebb0f9889e69","source_date_epoch":1785338401},"interpreter":{"cache_tag":"cpython-312","implementation":"cpython","version":"3.12.3"},"platform":{"machine":"x86_64","release":"6.17.0-1019-aws","sys_platform":"linux","system":"Linux"},"public_demo":{"classification":"synthetic_ietf_documentation_ranges_with_public_demo_key","commands":[{"argv":["netveil-audit","--version"],"exit_code":0,"stderr":"","stdout":"netveil-audit 0.3.0\n"},{"argv":["netveil-audit","receipt","documentation-corpus.txt","--key-file","public-demo.key"],"exit_code":0,"stderr":"","stdout_json":{"canonicalization":"netveil.sorted-keys-json.v1","protocol":"netveil.hmac-sha256-pseudonymization.v1","report":{"canonicalization":"netveil.sorted-keys-json.v1","counts":{"endpoint_occurrences":5,"physical_lines":6,"source_bytes":129,"unique_endpoints":4},"duplicates":{"extra_occurrences":1,"group_count":1,"groups":[{"extra_occurrences":1,"id":"nvd1_f382d197a21771fcdbc8da8d818d4612a6a6bf7d8ca45415298f475b5709da10","occurrences":2}]},"endpoint_occurrences_by_ip_version":{"ipv4":4,"ipv6":1},"endpoint_occurrences_by_port_bucket":{"dynamic_49152_65535":1,"registered_1024_49151":1,"system_1_1023":3},"endpoint_occurrences_by_scope":{"documentation":5,"global":0,"link_local":0,"loopback":0,"multicast":0,"private":0,"reserved":0,"shared":0,"site_local":0,"unspecified":0},"protocol":"netveil.hmac-sha256-pseudonymization.v1","runtime":{"endpoint_semantics":"python-stdlib-ipaddress","python_implementation":"cpython","python_version":"3.12.3","schema":"netveil.python-runtime.v1"},"schema":"netveil.aggregate-report.v1","source_content_id":"nvs1_db2e7cafc475293724cdaa15fd267fd750a46f33ea199091a14cfa7635523e31"},"report_digest":{"algorithm":"sha256","value":"e266ff9c6b81df76e9224772d67958be75b90b9a8ac67161182cf39985b76f1e"},"schema":"netveil.aggregate-receipt.v1"},"stdout_sha256":"de378c083090651dcc1805cc9e164ee3fa9789a9dfa7da72f921ad2cda879416"}],"corpus":{"physical_lines":6,"sha256":"49f7aab7d50edcf5a5bd2a9edc002e7c9303881817c15e8272e1e3ee1f36129e","size_bytes":129},"public_demo_key":{"classification":"public_non_secret_test_material","sha256":"2d27befbc438954c4a55d8c0e36192c5a4a7e9e3f15c17d6e427c4a3499d945d","size_bytes":32,"source_constant":"tools/verify_fresh_wheel.py:_PUBLIC_DEMO_KEY"}},"schema":"netveil.fresh-wheel-verification.v1","source_commit":"f3e4db737c4fd87b4b58f842a926ebb0f9889e69","status":"pass","syscall_traces":[{"exec_chain":["installed_launcher","installed_python"],"exec_count":2,"exit_syscall_count":1,"label":"receipt","network_syscall_count":0,"normalized_sha256":"0cdfc79931d31b9665f522437c58fb2df4b7a30cfdf69fccec996a206b35721a","post_launch_process_count":0,"process_count":1},{"exec_chain":["installed_launcher","installed_python"],"exec_count":2,"exit_syscall_count":1,"label":"version","network_syscall_count":0,"normalized_sha256":"f49c6be35c867adf767a07fcd01788478f57ecfddee88550f992f9275d3d7b57","post_launch_process_count":0,"process_count":1}],"wheel":{"members":[{"mode":"0600","path":"netveil/__init__.py","sha256":"ce9ae0bfd8d5080d58e86c661ddfe910d86894cd99ef7aca58053c4fdc5cfb6e","size_bytes":1216},{"mode":"0600","path":"netveil/cli.py","sha256":"577c91cf2a11aae46273e3ebe23ecab1f6e9834261de5b914d1644ac1552e243","size_bytes":11078},{"mode":"0600","path":"netveil/model.py","sha256":"1f333d4152ad8b956c971595300b49984a56dc9b9dd90e8cd32157f3fdffc4f3","size_bytes":6026},{"mode":"0600","path":"netveil/parser.py","sha256":"ee9c20174454cf8029432543d4358a71d77da0f837b13a3b375ef9d2d7179e8a","size_bytes":5106},{"mode":"0600","path":"netveil/privacy.py","sha256":"fba03e6d5a7d618797a0654252394af04f88069ddc5da13ada34199d62b5cbfd","size_bytes":18224},{"mode":"0644","path":"netveil/py.typed","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"mode":"0755","path":"netveil_audit-0.3.0.data/scripts/netveil-audit","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},{"mode":"0644","path":"netveil_audit-0.3.0.dist-info/METADATA","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"mode":"0664","path":"netveil_audit-0.3.0.dist-info/RECORD","sha256":"fcdee87a96f526d4d7546a733cf41b0898e917aa62b305a8b9ac637ab49df69d","size_bytes":1048},{"mode":"0600","path":"netveil_audit-0.3.0.dist-info/WHEEL","sha256":"2b6eb4118ce7cd7b09601406aa623c553c4476265836f0d9c16f5c061f7efcc0","size_bytes":91},{"mode":"0644","path":"netveil_audit-0.3.0.dist-info/licenses/LICENSE","sha256":"125e5c962ef12e9a92e39f9af7776d8010a98fce5f4559221be7033573c226a8","size_bytes":1069},{"mode":"0600","path":"netveil_audit-0.3.0.dist-info/top_level.txt","sha256":"9f7e05afb0b8dbbd49796a869258dc8cdc13dae0973dd848872799554d018ec7","size_bytes":26},{"mode":"0600","path":"netveil_bootstrap.py","sha256":"71205f05f3398b7ddc6ccf4b820c001f71f2932694f370dfccee2d31a33f9de8","size_bytes":16752}],"sha256":"0f4254362623a344e2474db96be81534f502e291b990fcf9ab1f0ba46d0f0675","size_bytes":26440}} diff --git a/docs/evidence/release-inventory.json b/docs/evidence/release-inventory.json new file mode 100644 index 0000000..13d2e9f --- /dev/null +++ b/docs/evidence/release-inventory.json @@ -0,0 +1 @@ +{"artifacts":[{"filename":"netveil_audit-0.3.0-py3-none-any.whl","kind":"wheel","sha256":"0f4254362623a344e2474db96be81534f502e291b990fcf9ab1f0ba46d0f0675","size_bytes":26440},{"filename":"netveil_audit-0.3.0.tar.gz","kind":"sdist","members":[{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/LICENSE","sha256":"125e5c962ef12e9a92e39f9af7776d8010a98fce5f4559221be7033573c226a8","size_bytes":1069},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/MANIFEST.in","sha256":"8fab5c0ebe83e4fe3ce4118045e2cbe48c70b4b60aabaadb9d06d335081f01d3","size_bytes":430},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/PKG-INFO","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/README.md","sha256":"97eb7a6405c8f2360d147fab133fab614e5fa661086a0b75bc3df2a7f62401b6","size_bytes":10824},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/SECURITY.md","sha256":"2e4f7ddfaf851ca804f2d9b6306cdda309646aa0dd87ed1f33ffec9745121201","size_bytes":8028},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/docs","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/artifact-boundary.md","sha256":"91b2026791ce76c3ab1662b79ea59649f9d6d31ab07aca7e0b4f80ba2c0361d5","size_bytes":8545},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/docs/assets","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/assets/architecture.svg","sha256":"5adc28a9088dc6f000ce810511726fe6d14cfcb6996fbff76a6e3f9154f28abc","size_bytes":8558},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/cli-contract.md","sha256":"6a1b32fed2e7fee8d0c44a1b2a1f70cb0153b63488ec9e7ca04371160484a59d","size_bytes":6541},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/docs/privacy-protocol.md","sha256":"d047daf39cb49b69ff9c486aa0a7e9e06e9d872eb8cadfa5e13eb3bd9e2804f1","size_bytes":5213},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/pyproject.toml","sha256":"af21a11f116f31b5f0783934e1583ac1984f4c8f7dea0512812168ae47248299","size_bytes":1445},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/requirements-dev.txt","sha256":"6ca7f1739fb5f044210252ca5e6893bc1fee5ceb30130aaa44731738d253cef4","size_bytes":348},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/scripts","size_bytes":0},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/scripts/netveil-audit","sha256":"056f2721f16c78e7fe0c24cf91e277be6ac9a71858756775c4c1f4df038dd3ac","size_bytes":8714},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/setup.cfg","sha256":"1c473cbaee8da5fc46e7f0158794af5cea4414c34a3cf3f180c2001f5e38bd3e","size_bytes":38},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src","size_bytes":0},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src/netveil","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/__init__.py","sha256":"ce9ae0bfd8d5080d58e86c661ddfe910d86894cd99ef7aca58053c4fdc5cfb6e","size_bytes":1216},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/cli.py","sha256":"577c91cf2a11aae46273e3ebe23ecab1f6e9834261de5b914d1644ac1552e243","size_bytes":11078},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/model.py","sha256":"1f333d4152ad8b956c971595300b49984a56dc9b9dd90e8cd32157f3fdffc4f3","size_bytes":6026},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/parser.py","sha256":"ee9c20174454cf8029432543d4358a71d77da0f837b13a3b375ef9d2d7179e8a","size_bytes":5106},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/privacy.py","sha256":"fba03e6d5a7d618797a0654252394af04f88069ddc5da13ada34199d62b5cbfd","size_bytes":18224},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil/py.typed","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/PKG-INFO","sha256":"d220705e247867f205f6f6df178a5380e2bbb05ef8117dbc3695dad1632a9a5d","size_bytes":11698},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/SOURCES.txt","sha256":"97e233d844643aa7fa463361c581cf46e4a6c6d0cc296c51758f8431d30ac231","size_bytes":958},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/dependency_links.txt","sha256":"01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b","size_bytes":1},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_audit.egg-info/top_level.txt","sha256":"9f7e05afb0b8dbbd49796a869258dc8cdc13dae0973dd848872799554d018ec7","size_bytes":26},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/src/netveil_bootstrap.py","sha256":"71205f05f3398b7ddc6ccf4b820c001f71f2932694f370dfccee2d31a33f9de8","size_bytes":16752},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tests","size_bytes":0},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tests/fixtures","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/fixtures/launcher_bootstrap_fixture.py","sha256":"ce44795d98e969bd625373e9e84064c013470d641e009989e569e68d2ef95b73","size_bytes":251},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_bootstrap.py","sha256":"9b9f18cff5cc8a3499a6617861e4e8e7817ecba8495637a9f663afc95e971ea0","size_bytes":40030},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_cli.py","sha256":"8f8ea9b0617262ef74dcd148f5933f39d74b067be4b54eefff3f438b4b68308e","size_bytes":27081},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_evidence_renderer.py","sha256":"76b4f6d9b4b9c350a1fcbdb32e7985ff64a2fc40c3c1f15622615235656b9c8c","size_bytes":11137},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_fresh_wheel_verifier.py","sha256":"0ed6c2d4f8c8ee65c5731005c7481b7c44655ae3fc8d9063fc455152e1f092ff","size_bytes":32914},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_launcher.py","sha256":"f9931cb6ec9e43ffc55ea9e89ee2188a3edede24df351ecd3c793a649a1f202e","size_bytes":21478},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_model.py","sha256":"fb54ea5687906b0410c0623626b2a16bc6b2b634bef17bb402811986b921b387","size_bytes":4737},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_packaging.py","sha256":"75eb1770f14b15c2baa4d8e1c3551c122d0de365a797d3849f4690e82de719d5","size_bytes":2603},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_parser.py","sha256":"bc606c4b0a7cb2c5fce74cba4133f9c625ea412c0d42670605e14e755d7305ab","size_bytes":10625},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_privacy.py","sha256":"d770c688d2dd13d0078c679dce4e2095c7a9c970a8e3c9a1d1194a7a779ad4f1","size_bytes":23160},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_release_builder.py","sha256":"b93c280b25f056c71d3c8038decc91185310d28f4af7bfb8a429b1f7c915a38e","size_bytes":20067},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tests/test_visuals.py","sha256":"5ec195c534d085b4ca59a578a54e177255da35d01a6deac419325dd786023df5","size_bytes":721},{"kind":"directory","mode":"0755","path":"netveil_audit-0.3.0/tools","size_bytes":0},{"kind":"file","mode":"0644","path":"netveil_audit-0.3.0/tools/__init__.py","sha256":"28db1b0a84cbcf8a228da02aae487a38895d51b66e2ea37067058252351b23db","size_bytes":55},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/build_release.py","sha256":"979ca44ad2121588c4ddf72d721f4d5d49449546a8f5a1f9e06796d39d9e820d","size_bytes":25873},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/render_architecture.py","sha256":"25f308aa838397bf3e331351ca6781267b49b43069e066d78ad4a3acf34ae741","size_bytes":9738},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/render_evidence.py","sha256":"9b00b1a58e61d92b899d86d9777f720a8565aafdfa42b25d34e2aa505d068172","size_bytes":42639},{"kind":"file","mode":"0755","path":"netveil_audit-0.3.0/tools/verify_fresh_wheel.py","sha256":"c88109e00821895446ff4ff71bef8a283af0c70fa723e9e2a7ed630ea369f6f6","size_bytes":79572}],"sha256":"7d1975194f8a439539f10e50f71b4f37da1dc335569d941e4f25990d4cb14139","size_bytes":107816}],"schema":"netveil.release-inventory.v1","source_commit":"f3e4db737c4fd87b4b58f842a926ebb0f9889e69","source_date_epoch":1785338401} diff --git a/docs/evidence/visual-manifest.json b/docs/evidence/visual-manifest.json new file mode 100644 index 0000000..493f4ea --- /dev/null +++ b/docs/evidence/visual-manifest.json @@ -0,0 +1 @@ +{"claim_boundary":"unsigned internally consistent build-and-execution evidence; not publisher authentication, host attestation, or a signature","generator":{"path":"tools/render_evidence.py","sha256":"1a14a0a66b1a56cd917eab0e15c87625e97a3849e47f30cd3d2985514617d3d9","size_bytes":42635},"inputs":[{"path":"docs/evidence/fresh-wheel-verification.json","sha256":"4938f49af70800713eab2c7734db149c605cad146c08940d55c7c0e0f6efc98d","size_bytes":16785},{"path":"docs/evidence/release-inventory.json","sha256":"e41685491e63cdfb8860ec7cce48ca1f8743da6d29cc7b5d3c398f738c1c6622","size_bytes":8609}],"outputs":[{"path":"docs/assets/artifact-provenance.svg","sha256":"40e22f1529c4123dfdb62322a926f36ee4206c2c6d655b8019c2ed9a1554a57a","size_bytes":4427},{"path":"docs/assets/cli-session.svg","sha256":"9a76e0d0511aa70ba7bcc703b40de62a76b93bd2459ef8bf26dd0d3f214b6120","size_bytes":4845},{"path":"docs/assets/receipt-counts.svg","sha256":"3f4d062c312f8534274f2dabd5c37bee10586e1dba34ce1cef2ba243389f1780","size_bytes":3620},{"path":"docs/assets/verification-matrix.svg","sha256":"af5bb1175f04cfd0654659e98a21f4ff3a3cb4c0525dd80f91a7e7d0735a6c62","size_bytes":7317},{"path":"docs/evidence/cli-session.cast","sha256":"4ebc2d4a493983a3490d5e0f9a04bcdaf117a2cb60ba60beadd27ad172ac8644","size_bytes":1805}],"schema":"netveil.visual-evidence.v1","source_commit":"f3e4db737c4fd87b4b58f842a926ebb0f9889e69"} diff --git a/docs/privacy-protocol.md b/docs/privacy-protocol.md new file mode 100644 index 0000000..11f71a3 --- /dev/null +++ b/docs/privacy-protocol.md @@ -0,0 +1,127 @@ +# Privacy report protocol v1 + +This document makes the byte-level Netveil v1 privacy-report contract +reproducible. It describes pseudonymization and aggregation, not anonymization. + +## Input boundary + +`build_privacy_report` and `build_privacy_receipt` accept: + +- an exact `bytes` endpoint corpus accepted by `parse_corpus`; and +- an exact `bytes` pseudonymization key containing at least 32 bytes. + +Parser size, line-count, UTF-8, syntax, and canonicalization bounds apply +unchanged. The report layer has no path, file, network, DNS, or process API. + +## Typed HMAC identifiers + +Both identifiers use: + +```text +HMAC-SHA256(key, domain || uint64be(value_length) || value) +``` + +`uint64be` is an unsigned eight-byte big-endian integer. Digest text is the 64 +lowercase hexadecimal characters returned by `HMAC.hexdigest()`. + +The exact, stable domains and values are: + +| Identifier | Typed prefix | Domain bytes as a Python literal | Value | +|---|---|---|---| +| source content | `nvs1_` | `b"netveil\x00hmac-sha256-pseudonymization\x00v1\x00source-content\x00"` | exact source bytes | +| duplicate group | `nvd1_` | `b"netveil\x00hmac-sha256-pseudonymization\x00v1\x00duplicate-group\x00"` | canonical endpoint encoded as ASCII | + +The HMAC domains cryptographically separate the purposes. The distinct typed +prefixes also prevent a source ID and duplicate-group ID from being +interpreted as the same semantic type. + +An endpoint ID is emitted only when that canonical endpoint occurs at least +twice. Equivalent IPv6 spellings are canonicalized before grouping. Duplicate +groups are ordered lexicographically by typed ID, never by raw endpoint or +input order. + +## Aggregate semantics + +Every categorical count is an endpoint-occurrence count, not a unique-value +count. Zero-count categories remain present. + +The fixed v1 scope labels are `documentation`, `global`, `private`, +`loopback`, `link_local`, `multicast`, `shared`, `site_local`, `unspecified`, +and `reserved`. Classification is inherited from the fail-closed parser. + +Port buckets are fixed for v1: + +- `system_1_1023`: ports 1 through 1023; +- `registered_1024_49151`: ports 1024 through 49151; +- `dynamic_49152_65535`: ports 49152 through 65535. + +`extra_occurrences` is the sum of `occurrences - 1` across duplicate groups. +`unique_endpoints + extra_occurrences` therefore equals +`endpoint_occurrences`. + +The public report also includes exact source byte size and parser physical-line +count. It never includes an ordinary digest of the source. + +## Canonical JSON and receipt binding + +`netveil.sorted-keys-json.v1` is the following restricted JSON profile: + +- UTF-8-compatible ASCII output from `ensure_ascii=True`; +- object keys sorted lexicographically at every level; +- compact `,` and `:` separators; +- no NaN or infinity; +- no trailing newline. + +The v1 schema contains integers, strings, objects, and arrays only, so number +or Unicode normalization ambiguity is not present. + +The report binds the runtime profile that supplied endpoint parsing, +canonicalization, and scope classification: + +- schema `netveil.python-runtime.v1`; +- `sys.implementation.name`; +- the exact Python `major.minor.micro` version; and +- endpoint semantics `python-stdlib-ipaddress`. + +The same payload and key are byte-deterministic under the same runtime profile +and exact protocol implementation. Python's stdlib `ipaddress` +classifications have changed between interpreter versions, so reports with +different runtime profiles are not claimed to be byte-identical. The profile +is inside the report-digest boundary rather than being ambient metadata. + +The receipt calculates: + +```text +report_sha256 = SHA256(canonical_public_report_bytes) +``` + +It then embeds both the report object and that lowercase digest. The digest +input is the standalone public report, which has no digest field; this avoids +self-reference. The digest is content-addressing metadata, not a signature or +proof of provenance. The receipt schema does not identify a source commit, +wheel digest, distribution version, interpreter executable hash, or external +evidence manifest. + +The public identifiers declare +`netveil.hmac-sha256-pseudonymization.v1`; the report and receipt declare +`netveil.aggregate-report.v1` and `netveil.aggregate-receipt.v1`. Any +incompatible change to domains, framing, canonicalization, aggregation, or +schema requires a new protocol/schema version. + +## Threat boundary + +HMAC prevents offline candidate testing only while the key remains secret. It +does not prevent guessing by someone who can submit chosen corpora to a report +generation oracle. Restrict access to both the key and any service that uses +it. + +Reusing a key deliberately exposes equality of exact source content and +duplicate groups across reports. Counts, sizes, categories, equality, and +frequency remain public and may identify a small corpus. Key rotation changes +the typed cryptographic IDs for independently generated keys, but aggregate +patterns can still permit inference. + +Python cannot guarantee zeroization of input bytes, keys, canonical endpoint +strings, or HMAC intermediates. Callers own key entropy, storage, access, +versioning, rotation, retirement, memory/process isolation, and output +disclosure decisions. diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..45653bb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,56 @@ +[build-system] +requires = ["setuptools==83.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "netveil-audit" +version = "0.3.0" +description = "Offline privacy-preserving audits for network-endpoint corpora" +readme = "README.md" +requires-python = ">=3.11" +license = "MIT" +license-files = ["LICENSE"] +authors = [{name = "Omar Ibrahim"}] +keywords = ["audit", "privacy", "provenance", "security", "supply-chain"] +classifiers = [ + "Development Status :: 4 - Beta", + "Environment :: Console", + "Operating System :: POSIX :: Linux", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Typing :: Typed", +] +dependencies = [] + +[project.urls] +Documentation = "https://github.com/omar07ibrahim/Hello-World#readme" +Issues = "https://github.com/omar07ibrahim/Hello-World/issues" +Repository = "https://github.com/omar07ibrahim/Hello-World" + +[tool.setuptools] +package-dir = {"" = "src"} +packages = ["netveil"] +py-modules = ["netveil_bootstrap"] +script-files = ["scripts/netveil-audit"] + +[tool.setuptools.package-data] +netveil = ["py.typed"] + +[tool.ruff] +line-length = 88 +target-version = "py311" + +[tool.mypy] +python_version = "3.11" +strict = true +files = ["src", "tests", "tools", "scripts/netveil-audit"] + +[tool.coverage.run] +branch = true +source = ["src", "scripts"] + +[tool.coverage.report] +fail_under = 99 +precision = 2 +show_missing = true diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..39fd5d5 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,14 @@ +# Exact tool versions used by the repository's local quality gate. +# Runtime dependencies remain empty; this file is development-only. +ast_serialize==0.6.0 +build==1.5.0 +coverage==7.15.2 +librt==0.13.0 +mypy==2.3.0 +mypy_extensions==1.1.0 +packaging==26.2 +pathspec==1.1.1 +pyproject_hooks==1.2.0 +ruff==0.16.0 +setuptools==83.0.0 +typing_extensions==4.16.0 diff --git a/scripts/netveil-audit b/scripts/netveil-audit new file mode 100755 index 0000000..2910244 --- /dev/null +++ b/scripts/netveil-audit @@ -0,0 +1,296 @@ +#!/bin/sh +""":" +case "$0" in + */*) netveil_script_directory=${0%/*} ;; + *) exit 70 ;; +esac +exec "$netveil_script_directory/python" -IESB "$0" "$@" +exit 70 +":""" + +"""Isolated installed-artifact launcher for Netveil.""" + +import base64 +import hashlib +import importlib.metadata +import os +import stat +import sys +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Final, NoReturn + +_DISTRIBUTION_NAME: Final = "netveil-audit" +_DISTRIBUTION_VERSION: Final = "0.3.0" +_BOOTSTRAP_FILE: Final = "netveil_bootstrap.py" +_LAUNCHER_NAME: Final = "netveil-audit" +_MAX_ARTIFACT_FILE_BYTES: Final = 1_048_576 +_ARTIFACT_FAILURE_EXIT: Final = 10 +_OUTPUT_FAILURE_EXIT: Final = 15 +_INTERNAL_FAILURE_EXIT: Final = 70 + + +class _LaunchFailure(Exception): + pass + + +def _fail() -> NoReturn: + raise _LaunchFailure + + +def _startup_is_isolated() -> bool: + flags = sys.flags + return ( + os.name == "posix" + and flags.isolated == 1 + and flags.no_site == 1 + and flags.ignore_environment == 1 + and flags.dont_write_bytecode == 1 + and flags.safe_path is True + and "" not in sys.path + ) + + +def _file_flags() -> int: + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if any(not hasattr(os, name) for name in required): + _fail() + return os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + + +def _identity(status: os.stat_result) -> tuple[int, ...]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_uid, + status.st_gid, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def _read_record_bound( + path: Path, + record: importlib.metadata.PackagePath, +) -> tuple[bytes, tuple[int, ...]]: + file_hash = record.hash + file_size = record.size + if ( + file_hash is None + or file_hash.mode != "sha256" + or type(file_size) is not int + or file_size < 0 + or file_size > _MAX_ARTIFACT_FILE_BYTES + ): + _fail() + + payload, identity = _read_bounded_regular(path) + digest = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()) + digest_text = digest.rstrip(b"=").decode("ascii") + if len(payload) != file_size or digest_text != file_hash.value: + _fail() + return payload, identity + + +def _read_bounded_regular(path: Path) -> tuple[bytes, tuple[int, ...]]: + descriptor = -1 + try: + descriptor = os.open(path, _file_flags()) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size < 0 + or before.st_size > _MAX_ARTIFACT_FILE_BYTES + ): + _fail() + chunks: list[bytes] = [] + observed = 0 + while observed <= before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail() + finally: + if descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + try: + os.close(descriptor) + except OSError: + _fail() + + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail() + return payload, _identity(after) + + +def _site_root(launcher_path: Path) -> Path: + if launcher_path.name != _LAUNCHER_NAME or launcher_path.parent.name != "bin": + _fail() + prefix = launcher_path.parent.parent + python_directory = f"python{sys.version_info.major}.{sys.version_info.minor}" + library_directories = tuple(dict.fromkeys(("lib", sys.platlibdir))) + candidates = tuple( + prefix / library / python_directory / package_directory + for library in library_directories + for package_directory in ("site-packages", "dist-packages") + ) + matches = tuple( + candidate + for candidate in candidates + if (candidate / f"netveil_audit-{_DISTRIBUTION_VERSION}.dist-info").is_dir() + ) + if len(matches) != 1: + _fail() + return matches[0] + + +def _records( + distribution: importlib.metadata.Distribution, +) -> dict[str, importlib.metadata.PackagePath]: + files = distribution.files + if files is None: + _fail() + records: dict[str, importlib.metadata.PackagePath] = {} + for record in files: + raw_path = str(record) + if raw_path in records: + _fail() + records[raw_path] = record + return records + + +def _verified_bootstrap( + launcher_path: Path, +) -> tuple[bytes, Path]: + site_root = _site_root(launcher_path) + dist_info = site_root / f"netveil_audit-{_DISTRIBUTION_VERSION}.dist-info" + _read_bounded_regular(dist_info / "METADATA") + _read_bounded_regular(dist_info / "RECORD") + try: + os.lstat(dist_info / "entry_points.txt") + except FileNotFoundError: + pass + except OSError: + _fail() + else: + _fail() + distribution = importlib.metadata.PathDistribution(dist_info) + try: + name = distribution.metadata["Name"] + version = distribution.metadata["Version"] + except KeyError: + _fail() + if name != _DISTRIBUTION_NAME or version != _DISTRIBUTION_VERSION: + _fail() + + records = _records(distribution) + launcher_record_path = PurePosixPath( + os.path.relpath(launcher_path, site_root) + ).as_posix() + launcher_record = records.get(launcher_record_path) + bootstrap_record = records.get(_BOOTSTRAP_FILE) + if launcher_record is None or bootstrap_record is None: + _fail() + + _, launcher_identity = _read_record_bound(launcher_path, launcher_record) + located_launcher = Path(str(distribution.locate_file(launcher_record))) + _, located_identity = _read_record_bound(located_launcher, launcher_record) + if launcher_identity != located_identity: + _fail() + + bootstrap_path = site_root / _BOOTSTRAP_FILE + payload, bootstrap_identity = _read_record_bound( + bootstrap_path, + bootstrap_record, + ) + located_bootstrap = Path(str(distribution.locate_file(bootstrap_record))) + _, located_bootstrap_identity = _read_record_bound( + located_bootstrap, + bootstrap_record, + ) + if bootstrap_identity != located_bootstrap_identity: + _fail() + + sys.path.append(str(site_root)) + return payload, bootstrap_path + + +def _execute_bootstrap(payload: bytes, bootstrap_path: Path) -> int: + forbidden = tuple( + name + for name in sys.modules + if name == "netveil_bootstrap" + or name == "netveil" + or name.startswith("netveil.") + ) + if forbidden: + _fail() + + module = ModuleType("netveil_bootstrap") + module.__file__ = str(bootstrap_path) + module.__package__ = "" + sys.modules[module.__name__] = module + try: + code = compile( + payload, + str(bootstrap_path), + "exec", + dont_inherit=True, + ) + exec(code, module.__dict__) # noqa: S102 - executes RECORD-bound bytes. + main = module.main + result = main(sys.argv[1:]) + except BaseException: + sys.modules.pop(module.__name__, None) + raise + if type(result) is not int: + _fail() + return result + + +def _write_failure(code: str, exit_code: int) -> int: + payload = f"netveil-audit: {code}\n" + offset = 0 + try: + while offset < len(payload): + written = sys.stderr.write(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return _OUTPUT_FAILURE_EXIT + offset += written + sys.stderr.flush() + except OSError: + return _OUTPUT_FAILURE_EXIT + return exit_code + + +def main() -> int: + try: + if not _startup_is_isolated(): + _fail() + launcher_path = Path(__file__).absolute() + payload, bootstrap_path = _verified_bootstrap(launcher_path) + return _execute_bootstrap(payload, bootstrap_path) + except _LaunchFailure: + return _write_failure("artifact_unverified", _ARTIFACT_FAILURE_EXIT) + except KeyboardInterrupt: + return _write_failure("interrupted", _INTERNAL_FAILURE_EXIT) + except Exception: # noqa: BLE001 - never expose launcher tracebacks. + return _write_failure("internal_error", _INTERNAL_FAILURE_EXIT) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/netveil/__init__.py b/src/netveil/__init__.py new file mode 100644 index 0000000..f2b3536 --- /dev/null +++ b/src/netveil/__init__.py @@ -0,0 +1,53 @@ +"""Offline endpoint parsing with redacted failures and no network access.""" + +from netveil.model import ( + Endpoint, + EndpointCorpus, + EndpointScope, + IPVersion, +) +from netveil.parser import ( + MAX_INPUT_BYTES, + MAX_PHYSICAL_LINES, + EndpointParseError, + EndpointParseErrorCode, + parse_corpus, +) +from netveil.privacy import ( + CANONICAL_JSON_PROTOCOL, + MIN_PSEUDONYMIZATION_KEY_BYTES, + PRIVACY_RECEIPT_SCHEMA, + PRIVACY_REPORT_SCHEMA, + PSEUDONYMIZATION_PROTOCOL, + RUNTIME_PROFILE_SCHEMA, + DuplicateGroup, + PrivacyReceipt, + PrivacyReport, + RuntimeProfile, + build_privacy_receipt, + build_privacy_report, +) + +__all__ = [ + "CANONICAL_JSON_PROTOCOL", + "MAX_INPUT_BYTES", + "MAX_PHYSICAL_LINES", + "MIN_PSEUDONYMIZATION_KEY_BYTES", + "PRIVACY_RECEIPT_SCHEMA", + "PRIVACY_REPORT_SCHEMA", + "PSEUDONYMIZATION_PROTOCOL", + "RUNTIME_PROFILE_SCHEMA", + "DuplicateGroup", + "Endpoint", + "EndpointCorpus", + "EndpointParseError", + "EndpointParseErrorCode", + "EndpointScope", + "IPVersion", + "PrivacyReceipt", + "PrivacyReport", + "RuntimeProfile", + "build_privacy_receipt", + "build_privacy_report", + "parse_corpus", +] diff --git a/src/netveil/cli.py b/src/netveil/cli.py new file mode 100644 index 0000000..33a321c --- /dev/null +++ b/src/netveil/cli.py @@ -0,0 +1,361 @@ +"""Fail-closed command-line boundary for installed Netveil artifacts.""" + +from __future__ import annotations + +import argparse +import hmac +import os +import stat +import sys +from collections.abc import Sequence +from dataclasses import dataclass +from enum import IntEnum +from pathlib import Path +from typing import Any, BinaryIO, Final, NoReturn, TextIO + +from netveil.parser import MAX_INPUT_BYTES, EndpointParseError, parse_corpus +from netveil.privacy import ( + MIN_PSEUDONYMIZATION_KEY_BYTES, + build_privacy_receipt, +) + +_DISTRIBUTION_NAME: Final = "netveil-audit" +_DISTRIBUTION_VERSION: Final = "0.3.0" +_MAX_KEY_BYTES: Final = 4_096 + + +class CliExitCode(IntEnum): + """Stable process exit codes for automation.""" + + SUCCESS = 0 + USAGE = 2 + ARTIFACT_UNVERIFIED = 10 + CORPUS_UNAVAILABLE = 11 + KEY_UNAVAILABLE = 12 + CORPUS_REJECTED = 13 + KEY_REJECTED = 14 + OUTPUT_FAILED = 15 + INTERNAL_ERROR = 70 + + +class _CliFailure(Exception): + def __init__(self, code: str, exit_code: CliExitCode) -> None: + super().__init__(code) + self.code = code + self.exit_code = exit_code + + +class _CliCompletion(Exception): + def __init__(self, exit_code: CliExitCode) -> None: + super().__init__(int(exit_code)) + self.exit_code = exit_code + + +@dataclass(frozen=True, slots=True) +class _ReadFile: + payload: bytes + device: int + inode: int + + +class _SafeArgumentParser(argparse.ArgumentParser): + def error(self, message: str) -> NoReturn: + del message + raise _CliFailure("usage_error", CliExitCode.USAGE) + + def _print_message( + self, + message: str | None, + file: Any | None = None, + ) -> None: + if message and not _write_text(file or sys.stderr, message): + raise _CliFailure("output_failed", CliExitCode.OUTPUT_FAILED) + + def exit(self, status: int = 0, message: str | None = None) -> NoReturn: + if message is not None: + self._print_message( + message, + sys.stdout if status == 0 else sys.stderr, + ) + if status == 0: + raise _CliCompletion(CliExitCode.SUCCESS) + raise _CliFailure("usage_error", CliExitCode.USAGE) + + +def _fail(code: str, exit_code: CliExitCode) -> NoReturn: + raise _CliFailure(code, exit_code) + + +def _open_flags() -> int: + required_flags = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if os.name != "posix" or any(not hasattr(os, name) for name in required_flags): + _fail("platform_unsupported", CliExitCode.INTERNAL_ERROR) + return os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + + +def _file_identity(status: os.stat_result) -> tuple[int, ...]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_uid, + status.st_gid, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def _read_open_file( + descriptor: int, + *, + maximum_bytes: int, +) -> tuple[bytes | None, os.stat_result | None, os.stat_result | None]: + before: os.stat_result | None = None + after: os.stat_result | None = None + payload: bytes | None = None + operation_failed = False + try: + before = os.fstat(descriptor) + if stat.S_ISREG(before.st_mode) and before.st_size <= maximum_bytes: + chunks: list[bytes] = [] + observed = 0 + while observed <= maximum_bytes: + chunk = os.read( + descriptor, + min(65_536, maximum_bytes + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + operation_failed = True + if operation_failed: + return None, None, None + return payload, before, after + + +def _read_bounded_file( + path: Path, + *, + maximum_bytes: int, + failure_code: str, + failure_exit: CliExitCode, + key_policy: bool, +) -> _ReadFile: + descriptor: int | None = None + open_failed = False + try: + descriptor = os.open(path, _open_flags()) + except OSError: + open_failed = True + if open_failed or descriptor is None: + _fail(failure_code, failure_exit) + + payload: bytes | None = None + before: os.stat_result | None = None + after: os.stat_result | None = None + close_failed = False + try: + payload, before, after = _read_open_file( + descriptor, + maximum_bytes=maximum_bytes, + ) + finally: + try: + os.close(descriptor) + except OSError: + close_failed = True + + if ( + close_failed + or payload is None + or before is None + or after is None + or not stat.S_ISREG(before.st_mode) + or len(payload) > maximum_bytes + or len(payload) != after.st_size + or _file_identity(before) != _file_identity(after) + ): + _fail(failure_code, failure_exit) + + if key_policy and ( + len(payload) < MIN_PSEUDONYMIZATION_KEY_BYTES + or before.st_uid != os.geteuid() + or before.st_nlink != 1 + or not before.st_mode & stat.S_IRUSR + or before.st_mode + & ( + stat.S_IRWXG + | stat.S_IRWXO + | stat.S_IXUSR + | stat.S_ISUID + | stat.S_ISGID + | stat.S_ISVTX + ) + ): + _fail("key_rejected", CliExitCode.KEY_REJECTED) + return _ReadFile( + payload=payload, + device=before.st_dev, + inode=before.st_ino, + ) + + +def _parser(version: str) -> _SafeArgumentParser: + parser = _SafeArgumentParser( + prog="netveil-audit", + description="Create an offline pseudonymized endpoint-corpus receipt.", + allow_abbrev=False, + ) + parser.add_argument( + "--version", + action="store_true", + help="print the verified installed distribution version", + ) + subparsers = parser.add_subparsers(dest="command") + receipt = subparsers.add_parser( + "receipt", + help="write one canonical public receipt to standard output", + allow_abbrev=False, + ) + receipt.add_argument("corpus", type=Path, metavar="CORPUS") + receipt.add_argument( + "--key-file", + required=True, + type=Path, + metavar="KEY_FILE", + ) + parser.set_defaults(distribution_version=version) + return parser + + +def _write_text(stream: TextIO, payload: str) -> bool: + offset = 0 + try: + while offset < len(payload): + written = stream.write(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return False + offset += written + stream.flush() + except OSError: + return False + return True + + +def _write_binary(stream: BinaryIO, payload: bytes) -> bool: + offset = 0 + try: + while offset < len(payload): + written = stream.write(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return False + offset += written + stream.flush() + except (BrokenPipeError, OSError): + return False + return True + + +def _emit_failure(failure: _CliFailure) -> int: + if not _write_text(sys.stderr, f"netveil-audit: {failure.code}\n"): + return int(CliExitCode.OUTPUT_FAILED) + return int(failure.exit_code) + + +def _write_receipt(payload: bytes) -> None: + if not _write_binary(sys.stdout.buffer, payload + b"\n"): + _fail("output_failed", CliExitCode.OUTPUT_FAILED) + + +def _run_receipt(corpus_path: Path, key_path: Path) -> None: + corpus_file = _read_bounded_file( + corpus_path, + maximum_bytes=MAX_INPUT_BYTES, + failure_code="corpus_unavailable", + failure_exit=CliExitCode.CORPUS_UNAVAILABLE, + key_policy=False, + ) + parse_failure: str | None = None + try: + parse_corpus(corpus_file.payload) + except EndpointParseError as error: + location = "" if error.line_number is None else f":line={error.line_number}" + parse_failure = f"corpus_rejected:{error.code.value}{location}" + if parse_failure is not None: + _fail( + parse_failure, + CliExitCode.CORPUS_REJECTED, + ) + + key_file = _read_bounded_file( + key_path, + maximum_bytes=_MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + if (corpus_file.device, corpus_file.inode) == (key_file.device, key_file.inode) or ( + len(corpus_file.payload) == len(key_file.payload) + and hmac.compare_digest(corpus_file.payload, key_file.payload) + ): + _fail("key_rejected", CliExitCode.KEY_REJECTED) + receipt = build_privacy_receipt( + corpus_file.payload, + pseudonymization_key=key_file.payload, + ) + _write_receipt(receipt.canonical_json_bytes()) + + +def _main_verified( + argv: Sequence[str] | None = None, + *, + verified_distribution_version: str | None = None, +) -> int: + """Run only after the external bootstrap has verified source bytes.""" + + try: + if verified_distribution_version != _DISTRIBUTION_VERSION: + _fail("artifact_unverified", CliExitCode.ARTIFACT_UNVERIFIED) + raw_arguments = tuple(sys.argv[1:] if argv is None else argv) + if "--version" in raw_arguments and raw_arguments != ("--version",): + _fail("usage_error", CliExitCode.USAGE) + key_option_count = sum( + argument == "--key-file" or argument.startswith("--key-file=") + for argument in raw_arguments + ) + if key_option_count > 1 or "--key-file=" in raw_arguments: + _fail("usage_error", CliExitCode.USAGE) + arguments = _parser(verified_distribution_version).parse_args(raw_arguments) + if arguments.version: + if not _write_text( + sys.stdout, + f"{_DISTRIBUTION_NAME} {verified_distribution_version}\n", + ): + _fail("output_failed", CliExitCode.OUTPUT_FAILED) + return int(CliExitCode.SUCCESS) + if arguments.command != "receipt": + _fail("usage_error", CliExitCode.USAGE) + _run_receipt(arguments.corpus, arguments.key_file) + except _CliCompletion as completion: + return int(completion.exit_code) + except _CliFailure as failure: + return _emit_failure(failure) + except KeyboardInterrupt: + return _emit_failure(_CliFailure("interrupted", CliExitCode.INTERNAL_ERROR)) + except Exception: # noqa: BLE001 - CLI must never render sensitive tracebacks. + return _emit_failure(_CliFailure("internal_error", CliExitCode.INTERNAL_ERROR)) + return int(CliExitCode.SUCCESS) diff --git a/src/netveil/model.py b/src/netveil/model.py new file mode 100644 index 0000000..0c33914 --- /dev/null +++ b/src/netveil/model.py @@ -0,0 +1,188 @@ +"""Immutable endpoint corpus model with internal validated construction.""" + +from __future__ import annotations + +import hashlib +import ipaddress +from dataclasses import dataclass +from enum import Enum +from typing import Self, final + +_DOCUMENTATION_NETWORKS = ( + ipaddress.ip_network("192.0.2.0/24"), + ipaddress.ip_network("198.51.100.0/24"), + ipaddress.ip_network("203.0.113.0/24"), + ipaddress.ip_network("2001:db8::/32"), +) +_SHARED_ADDRESS_SPACE = ipaddress.ip_network("100.64.0.0/10") + + +class IPVersion(Enum): + """Supported IP address families.""" + + IPV4 = 4 + IPV6 = 6 + + +class EndpointScope(Enum): + """Privacy-relevant address classification.""" + + DOCUMENTATION = "documentation" + GLOBAL = "global" + PRIVATE = "private" + LOOPBACK = "loopback" + LINK_LOCAL = "link_local" + MULTICAST = "multicast" + SHARED = "shared" + SITE_LOCAL = "site_local" + UNSPECIFIED = "unspecified" + RESERVED = "reserved" + + +@final +@dataclass(frozen=True, slots=True, init=False) +class Endpoint: + """One canonical raw IP endpoint produced only by ``parse_corpus``.""" + + address: str + port: int + version: IPVersion + scope: EndpointScope + + def __new__(cls) -> Self: + """Block direct construction outside the validated parser boundary.""" + + raise TypeError("Endpoint objects are created by parse_corpus") + + @property + def canonical(self) -> str: + """Return the unambiguous canonical endpoint representation.""" + + if self.version is IPVersion.IPV6: + return f"[{self.address}]:{self.port}" + return f"{self.address}:{self.port}" + + +@final +@dataclass(frozen=True, slots=True, init=False) +class EndpointCorpus: + """A parsed corpus bound to exact bytes by the internal factory.""" + + source_sha256: str + source_bytes: int + physical_line_count: int + endpoints: tuple[Endpoint, ...] + unique_endpoints: tuple[Endpoint, ...] + + def __new__(cls) -> Self: + """Block direct construction outside the validated parser boundary.""" + + raise TypeError("EndpointCorpus objects are created by parse_corpus") + + @property + def endpoint_count(self) -> int: + return len(self.endpoints) + + @property + def unique_count(self) -> int: + return len(self.unique_endpoints) + + @property + def duplicate_count(self) -> int: + return self.endpoint_count - self.unique_count + + +def _scope( + address: ipaddress.IPv4Address | ipaddress.IPv6Address, +) -> EndpointScope: + """Classify one exact stdlib address without any network operation.""" + + if any(address in network for network in _DOCUMENTATION_NETWORKS): + return EndpointScope.DOCUMENTATION + if address.is_unspecified: + return EndpointScope.UNSPECIFIED + if address.is_loopback: + return EndpointScope.LOOPBACK + if address.is_multicast: + return EndpointScope.MULTICAST + if address.is_link_local: + return EndpointScope.LINK_LOCAL + if isinstance(address, ipaddress.IPv6Address) and address.is_site_local: + return EndpointScope.SITE_LOCAL + if address in _SHARED_ADDRESS_SPACE: + return EndpointScope.SHARED + if address.is_reserved: + return EndpointScope.RESERVED + if address.is_private: + return EndpointScope.PRIVATE + return EndpointScope.GLOBAL + + +def _create_endpoint( + address: ipaddress.IPv4Address | ipaddress.IPv6Address, + *, + port: int, +) -> Endpoint: + """Create an endpoint after checking every stored field invariant.""" + + if type(address) not in (ipaddress.IPv4Address, ipaddress.IPv6Address): + raise TypeError("address must be an exact IPv4Address or IPv6Address") + if type(port) is not int: + raise TypeError("port must be an exact int") + if not 1 <= port <= 65_535: + raise ValueError("port must be in the range 1..65535") + if isinstance(address, ipaddress.IPv6Address) and address.scope_id is not None: + raise ValueError("scoped IPv6 addresses are not supported") + + endpoint: Endpoint = object.__new__(Endpoint) + object.__setattr__(endpoint, "address", address.compressed) + object.__setattr__(endpoint, "port", port) + object.__setattr__( + endpoint, + "version", + IPVersion.IPV6 + if isinstance(address, ipaddress.IPv6Address) + else IPVersion.IPV4, + ) + object.__setattr__(endpoint, "scope", _scope(address)) + return endpoint + + +def _create_corpus( + *, + payload: bytes, + physical_line_count: int, + endpoints: tuple[Endpoint, ...], +) -> EndpointCorpus: + """Create a corpus while deriving all identity and uniqueness fields.""" + + if type(payload) is not bytes: + raise TypeError("payload must be exact bytes") + if type(physical_line_count) is not int: + raise TypeError("physical_line_count must be an exact int") + if type(endpoints) is not tuple: + raise TypeError("endpoints must be an exact tuple") + if physical_line_count < 0: + raise ValueError("physical_line_count must be non-negative") + if not endpoints: + raise ValueError("a corpus must contain at least one endpoint") + if physical_line_count < len(endpoints): + raise ValueError("physical_line_count cannot be smaller than endpoint_count") + if any(type(endpoint) is not Endpoint for endpoint in endpoints): + raise TypeError("every endpoint must come from the validated factory") + + unique_by_canonical: dict[str, Endpoint] = {} + for endpoint in endpoints: + unique_by_canonical.setdefault(endpoint.canonical, endpoint) + + corpus: EndpointCorpus = object.__new__(EndpointCorpus) + object.__setattr__(corpus, "source_sha256", hashlib.sha256(payload).hexdigest()) + object.__setattr__(corpus, "source_bytes", len(payload)) + object.__setattr__(corpus, "physical_line_count", physical_line_count) + object.__setattr__(corpus, "endpoints", endpoints) + object.__setattr__( + corpus, + "unique_endpoints", + tuple(unique_by_canonical.values()), + ) + return corpus diff --git a/src/netveil/parser.py b/src/netveil/parser.py new file mode 100644 index 0000000..4013b01 --- /dev/null +++ b/src/netveil/parser.py @@ -0,0 +1,162 @@ +"""Fail-closed parser for local IP endpoint corpora.""" + +from __future__ import annotations + +import ipaddress +import re +from contextlib import suppress +from enum import Enum +from typing import NoReturn + +from netveil.model import ( + Endpoint, + EndpointCorpus, + _create_corpus, + _create_endpoint, +) + +MAX_INPUT_BYTES = 8 * 1024 * 1024 +MAX_PHYSICAL_LINES = 100_000 + +_IPV4_ENDPOINT = re.compile(r"^(?P
[^:[\]]+):(?P[0-9]+)$") +_IPV6_ENDPOINT = re.compile(r"^\[(?P
[^\[\]]+)\]:(?P[0-9]+)$") +_NON_LF_LINE_SEPARATORS = frozenset( + { + "\v", + "\f", + "\x1c", + "\x1d", + "\x1e", + "\x85", + "\u2028", + "\u2029", + } +) + + +class EndpointParseErrorCode(Enum): + """Stable public error codes that do not expose source values.""" + + EMPTY_CORPUS = "empty_corpus" + INPUT_TOO_LARGE = "input_too_large" + INVALID_ADDRESS = "invalid_address" + INVALID_LINE_ENDING = "invalid_line_ending" + INVALID_PORT = "invalid_port" + INVALID_SYNTAX = "invalid_syntax" + INVALID_UTF8 = "invalid_utf8" + NON_CANONICAL_PORT = "non_canonical_port" + TOO_MANY_LINES = "too_many_lines" + + +class EndpointParseError(ValueError): + """A redacted parse failure.""" + + def __init__( + self, + code: EndpointParseErrorCode, + *, + line_number: int | None = None, + ) -> None: + self.code = code + self.line_number = line_number + location = "" if line_number is None else f" at line {line_number}" + super().__init__(f"endpoint corpus rejected: {code.value}{location}") + + +def _fail( + code: EndpointParseErrorCode, + *, + line_number: int | None = None, +) -> NoReturn: + raise EndpointParseError(code, line_number=line_number) + + +def _parse_port(raw_port: str, *, line_number: int) -> int: + if len(raw_port) > 1 and raw_port.startswith("0"): + _fail(EndpointParseErrorCode.NON_CANONICAL_PORT, line_number=line_number) + if len(raw_port) > 5: + _fail(EndpointParseErrorCode.INVALID_PORT, line_number=line_number) + port = int(raw_port) + if not 1 <= port <= 65_535: + _fail(EndpointParseErrorCode.INVALID_PORT, line_number=line_number) + return port + + +def _parse_endpoint(raw_line: str, *, line_number: int) -> Endpoint: + if raw_line != raw_line.strip() or any( + character.isspace() for character in raw_line + ): + _fail(EndpointParseErrorCode.INVALID_SYNTAX, line_number=line_number) + + ipv6_match = _IPV6_ENDPOINT.fullmatch(raw_line) + ipv4_match = _IPV4_ENDPOINT.fullmatch(raw_line) + if ipv6_match is not None: + match = ipv6_match + elif ipv4_match is not None: + match = ipv4_match + else: + _fail(EndpointParseErrorCode.INVALID_SYNTAX, line_number=line_number) + + raw_address = match.group("address") + raw_port = match.group("port") + if "%" in raw_address: + _fail(EndpointParseErrorCode.INVALID_ADDRESS, line_number=line_number) + parsed_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None + with suppress(ValueError): + parsed_address = ipaddress.ip_address(raw_address) + if parsed_address is None: + _fail(EndpointParseErrorCode.INVALID_ADDRESS, line_number=line_number) + + if (ipv6_match is not None) != isinstance( + parsed_address, + ipaddress.IPv6Address, + ): + _fail(EndpointParseErrorCode.INVALID_SYNTAX, line_number=line_number) + + return _create_endpoint( + parsed_address, + port=_parse_port(raw_port, line_number=line_number), + ) + + +def parse_corpus(payload: bytes) -> EndpointCorpus: + """Parse exact local bytes without resolving or contacting any endpoint.""" + + if type(payload) is not bytes: + raise TypeError("payload must be exact bytes") + if len(payload) > MAX_INPUT_BYTES: + _fail(EndpointParseErrorCode.INPUT_TOO_LARGE) + if b"\r" in payload.replace(b"\r\n", b""): + _fail(EndpointParseErrorCode.INVALID_LINE_ENDING) + text: str | None = None + with suppress(UnicodeDecodeError): + text = payload.decode("utf-8", errors="strict") + if text is None: + _fail(EndpointParseErrorCode.INVALID_UTF8) + if any(character in _NON_LF_LINE_SEPARATORS for character in text): + _fail(EndpointParseErrorCode.INVALID_LINE_ENDING) + + normalized_text = text.replace("\r\n", "\n") + if normalized_text: + physical_lines = normalized_text.split("\n") + if physical_lines[-1] == "": + physical_lines.pop() + else: + physical_lines = [] + if len(physical_lines) > MAX_PHYSICAL_LINES: + _fail(EndpointParseErrorCode.TOO_MANY_LINES) + + endpoints: list[Endpoint] = [] + for line_number, raw_line in enumerate(physical_lines, start=1): + if not raw_line or raw_line.startswith("#"): + continue + endpoints.append(_parse_endpoint(raw_line, line_number=line_number)) + + if not endpoints: + _fail(EndpointParseErrorCode.EMPTY_CORPUS) + + return _create_corpus( + payload=payload, + physical_line_count=len(physical_lines), + endpoints=tuple(endpoints), + ) diff --git a/src/netveil/privacy.py b/src/netveil/privacy.py new file mode 100644 index 0000000..882db9a --- /dev/null +++ b/src/netveil/privacy.py @@ -0,0 +1,517 @@ +"""Privacy-preserving aggregate reports for endpoint corpora. + +This module deliberately keeps raw endpoint models on the private side of the +report boundary. Public models contain only keyed identifiers and aggregate +counts. +""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import sys +from collections import Counter +from dataclasses import dataclass +from typing import Final, Self, final + +from netveil.model import EndpointCorpus, EndpointScope, IPVersion +from netveil.parser import MAX_INPUT_BYTES, MAX_PHYSICAL_LINES, parse_corpus + +MIN_PSEUDONYMIZATION_KEY_BYTES: Final = 32 +PRIVACY_REPORT_SCHEMA: Final = "netveil.aggregate-report.v1" +PRIVACY_RECEIPT_SCHEMA: Final = "netveil.aggregate-receipt.v1" +PSEUDONYMIZATION_PROTOCOL: Final = "netveil.hmac-sha256-pseudonymization.v1" +CANONICAL_JSON_PROTOCOL: Final = "netveil.sorted-keys-json.v1" +RUNTIME_PROFILE_SCHEMA: Final = "netveil.python-runtime.v1" + +_SOURCE_CONTENT_DOMAIN: Final = ( + b"netveil\x00hmac-sha256-pseudonymization\x00v1\x00source-content\x00" +) +_DUPLICATE_GROUP_DOMAIN: Final = ( + b"netveil\x00hmac-sha256-pseudonymization\x00v1\x00duplicate-group\x00" +) +_SOURCE_CONTENT_ID_PREFIX: Final = "nvs1_" +_DUPLICATE_GROUP_ID_PREFIX: Final = "nvd1_" +_HEX_DIGITS: Final = frozenset("0123456789abcdef") +_MIN_CANONICAL_ENDPOINT_BYTES: Final = len(b"[::]:1") + +_IP_VERSION_LABELS: Final = ("ipv4", "ipv6") +_SCOPE_LABELS: Final = tuple(scope.value for scope in EndpointScope) +_PORT_BUCKET_LABELS: Final = ( + "system_1_1023", + "registered_1024_49151", + "dynamic_49152_65535", +) + +CountPairs = tuple[tuple[str, int], ...] + + +@final +@dataclass(frozen=True, slots=True, init=False) +class RuntimeProfile: + """The runtime whose stdlib defines endpoint parsing and classification.""" + + schema: str + python_implementation: str + python_version: str + endpoint_semantics: str + + def __new__(cls) -> Self: + """Block construction outside the report factory.""" + + raise TypeError("RuntimeProfile objects are created by build_privacy_report") + + +@final +@dataclass(frozen=True, slots=True, init=False) +class DuplicateGroup: + """One pseudonymous endpoint value that occurs more than once.""" + + group_id: str + occurrences: int + + def __new__(cls) -> Self: + """Block construction outside the validated report factory.""" + + raise TypeError("DuplicateGroup objects are created by build_privacy_report") + + @property + def extra_occurrences(self) -> int: + """Return occurrences beyond the first instance.""" + + return self.occurrences - 1 + + +@final +@dataclass(frozen=True, slots=True, init=False) +class PrivacyReport: + """Immutable public aggregates with no raw endpoint or source digest.""" + + source_content_id: str + runtime_profile: RuntimeProfile + source_bytes: int + physical_line_count: int + endpoint_count: int + unique_endpoint_count: int + ip_version_counts: CountPairs + scope_counts: CountPairs + port_bucket_counts: CountPairs + duplicate_groups: tuple[DuplicateGroup, ...] + + def __new__(cls) -> Self: + """Block construction outside the validated report factory.""" + + raise TypeError("PrivacyReport objects are created by build_privacy_report") + + @property + def duplicate_group_count(self) -> int: + """Return the number of distinct endpoint values with duplicates.""" + + return len(self.duplicate_groups) + + @property + def duplicate_occurrence_count(self) -> int: + """Return total endpoint occurrences beyond each group's first.""" + + return sum(group.extra_occurrences for group in self.duplicate_groups) + + def canonical_json_bytes(self) -> bytes: + """Serialize the public report using Netveil's canonical JSON profile.""" + + return _canonical_json_bytes(_report_document(self)) + + +@final +@dataclass(frozen=True, slots=True, init=False) +class PrivacyReceipt: + """A report plus a digest of the report's canonical public bytes.""" + + report: PrivacyReport + report_sha256: str + + def __new__(cls) -> Self: + """Block construction outside the validated receipt factory.""" + + raise TypeError("PrivacyReceipt objects are created by build_privacy_receipt") + + def canonical_json_bytes(self) -> bytes: + """Serialize the receipt without including its digest input recursively.""" + + document = { + "canonicalization": CANONICAL_JSON_PROTOCOL, + "protocol": PSEUDONYMIZATION_PROTOCOL, + "report": _report_document(self.report), + "report_digest": { + "algorithm": "sha256", + "value": self.report_sha256, + }, + "schema": PRIVACY_RECEIPT_SCHEMA, + } + return _canonical_json_bytes(document) + + +def _canonical_json_bytes(document: object) -> bytes: + return json.dumps( + document, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def _count_document(counts: CountPairs) -> dict[str, int]: + return dict(counts) + + +def _report_document(report: PrivacyReport) -> dict[str, object]: + return { + "canonicalization": CANONICAL_JSON_PROTOCOL, + "counts": { + "endpoint_occurrences": report.endpoint_count, + "physical_lines": report.physical_line_count, + "source_bytes": report.source_bytes, + "unique_endpoints": report.unique_endpoint_count, + }, + "duplicates": { + "extra_occurrences": report.duplicate_occurrence_count, + "group_count": report.duplicate_group_count, + "groups": [ + { + "extra_occurrences": group.extra_occurrences, + "id": group.group_id, + "occurrences": group.occurrences, + } + for group in report.duplicate_groups + ], + }, + "endpoint_occurrences_by_ip_version": _count_document(report.ip_version_counts), + "endpoint_occurrences_by_port_bucket": _count_document( + report.port_bucket_counts + ), + "endpoint_occurrences_by_scope": _count_document(report.scope_counts), + "protocol": PSEUDONYMIZATION_PROTOCOL, + "runtime": { + "endpoint_semantics": report.runtime_profile.endpoint_semantics, + "python_implementation": report.runtime_profile.python_implementation, + "python_version": report.runtime_profile.python_version, + "schema": report.runtime_profile.schema, + }, + "schema": PRIVACY_REPORT_SCHEMA, + "source_content_id": report.source_content_id, + } + + +def _keyed_identifier( + key: bytes, + *, + domain: bytes, + value: bytes, + prefix: str, +) -> str: + framed_value = len(value).to_bytes(8, byteorder="big") + value + digest = hmac.new(key, domain + framed_value, hashlib.sha256).hexdigest() + return prefix + digest + + +def _validate_pseudonymization_key(key: bytes) -> None: + if type(key) is not bytes: + raise TypeError("pseudonymization_key must be exact bytes") + if len(key) < MIN_PSEUDONYMIZATION_KEY_BYTES: + raise ValueError( + "pseudonymization_key must contain at least " + f"{MIN_PSEUDONYMIZATION_KEY_BYTES} bytes" + ) + + +def _validate_identifier(value: str, *, prefix: str, field: str) -> None: + if type(value) is not str: + raise TypeError(f"{field} must be an exact str") + suffix = value.removeprefix(prefix) + if ( + not value.startswith(prefix) + or len(suffix) != hashlib.sha256().digest_size * 2 + or any(character not in _HEX_DIGITS for character in suffix) + ): + raise ValueError(f"{field} must be a valid typed HMAC-SHA256 identifier") + + +def _validate_exact_nonnegative_int(value: int, *, field: str) -> None: + if type(value) is not int: + raise TypeError(f"{field} must be an exact int") + if value < 0: + raise ValueError(f"{field} must be non-negative") + + +def _validate_count_pairs( + counts: CountPairs, + *, + field: str, + labels: tuple[str, ...], + expected_total: int, +) -> None: + if type(counts) is not tuple: + raise TypeError(f"{field} must be an exact tuple") + for item in counts: + if type(item) is not tuple or len(item) != 2: + raise TypeError(f"{field} entries must be exact two-item tuples") + label, count = item + if type(label) is not str: + raise TypeError(f"{field} labels must be exact strings") + _validate_exact_nonnegative_int(count, field=f"{field} count") + if tuple(label for label, _ in counts) != labels: + raise ValueError(f"{field} labels or order are invalid") + if sum(count for _, count in counts) != expected_total: + raise ValueError(f"{field} must sum to endpoint_count") + + +def _create_duplicate_group(*, group_id: str, occurrences: int) -> DuplicateGroup: + _validate_identifier( + group_id, + prefix=_DUPLICATE_GROUP_ID_PREFIX, + field="group_id", + ) + _validate_exact_nonnegative_int(occurrences, field="occurrences") + if occurrences < 2: + raise ValueError("duplicate group occurrences must be at least two") + + group: DuplicateGroup = object.__new__(DuplicateGroup) + object.__setattr__(group, "group_id", group_id) + object.__setattr__(group, "occurrences", occurrences) + return group + + +def _create_runtime_profile() -> RuntimeProfile: + version = sys.version_info + profile: RuntimeProfile = object.__new__(RuntimeProfile) + object.__setattr__(profile, "schema", RUNTIME_PROFILE_SCHEMA) + object.__setattr__(profile, "python_implementation", sys.implementation.name) + object.__setattr__( + profile, + "python_version", + f"{version.major}.{version.minor}.{version.micro}", + ) + object.__setattr__( + profile, + "endpoint_semantics", + "python-stdlib-ipaddress", + ) + return profile + + +def _validate_runtime_profile(profile: RuntimeProfile) -> None: + if type(profile) is not RuntimeProfile: + raise TypeError("runtime_profile must come from the validated factory") + if profile != _create_runtime_profile(): + raise ValueError("runtime_profile does not match the active runtime") + + +def _create_privacy_report( + *, + source_content_id: str, + runtime_profile: RuntimeProfile, + source_bytes: int, + physical_line_count: int, + endpoint_count: int, + unique_endpoint_count: int, + ip_version_counts: CountPairs, + scope_counts: CountPairs, + port_bucket_counts: CountPairs, + duplicate_groups: tuple[DuplicateGroup, ...], +) -> PrivacyReport: + _validate_identifier( + source_content_id, + prefix=_SOURCE_CONTENT_ID_PREFIX, + field="source_content_id", + ) + _validate_runtime_profile(runtime_profile) + for field, value in ( + ("source_bytes", source_bytes), + ("physical_line_count", physical_line_count), + ("endpoint_count", endpoint_count), + ("unique_endpoint_count", unique_endpoint_count), + ): + _validate_exact_nonnegative_int(value, field=field) + if endpoint_count == 0: + raise ValueError("endpoint_count must be positive") + if source_bytes == 0: + raise ValueError("source_bytes must be positive") + if source_bytes > MAX_INPUT_BYTES: + raise ValueError("source_bytes exceeds the parser input bound") + if physical_line_count > MAX_PHYSICAL_LINES: + raise ValueError("physical_line_count exceeds the parser line bound") + if not 1 <= unique_endpoint_count <= endpoint_count: + raise ValueError("unique_endpoint_count must be in 1..endpoint_count") + if physical_line_count < endpoint_count: + raise ValueError("physical_line_count cannot be smaller than endpoint_count") + minimum_source_bytes = ( + endpoint_count * _MIN_CANONICAL_ENDPOINT_BYTES + physical_line_count - 1 + ) + if source_bytes < minimum_source_bytes: + raise ValueError("source_bytes is too small for the aggregate line counts") + + _validate_count_pairs( + ip_version_counts, + field="ip_version_counts", + labels=_IP_VERSION_LABELS, + expected_total=endpoint_count, + ) + _validate_count_pairs( + scope_counts, + field="scope_counts", + labels=_SCOPE_LABELS, + expected_total=endpoint_count, + ) + _validate_count_pairs( + port_bucket_counts, + field="port_bucket_counts", + labels=_PORT_BUCKET_LABELS, + expected_total=endpoint_count, + ) + if type(duplicate_groups) is not tuple: + raise TypeError("duplicate_groups must be an exact tuple") + if any(type(group) is not DuplicateGroup for group in duplicate_groups): + raise TypeError("every duplicate group must come from the validated factory") + for group in duplicate_groups: + _validate_identifier( + group.group_id, + prefix=_DUPLICATE_GROUP_ID_PREFIX, + field="group_id", + ) + _validate_exact_nonnegative_int(group.occurrences, field="occurrences") + if group.occurrences < 2: + raise ValueError("duplicate group occurrences must be at least two") + group_ids = tuple(group.group_id for group in duplicate_groups) + if group_ids != tuple(sorted(group_ids)) or len(group_ids) != len(set(group_ids)): + raise ValueError("duplicate_groups must have unique IDs in sorted order") + if len(duplicate_groups) > unique_endpoint_count: + raise ValueError("duplicate_group_count cannot exceed unique_endpoint_count") + expected_duplicate_count = endpoint_count - unique_endpoint_count + if ( + sum(group.extra_occurrences for group in duplicate_groups) + != expected_duplicate_count + ): + raise ValueError("duplicate_groups do not match aggregate uniqueness counts") + + report: PrivacyReport = object.__new__(PrivacyReport) + object.__setattr__(report, "source_content_id", source_content_id) + object.__setattr__(report, "runtime_profile", runtime_profile) + object.__setattr__(report, "source_bytes", source_bytes) + object.__setattr__(report, "physical_line_count", physical_line_count) + object.__setattr__(report, "endpoint_count", endpoint_count) + object.__setattr__(report, "unique_endpoint_count", unique_endpoint_count) + object.__setattr__(report, "ip_version_counts", ip_version_counts) + object.__setattr__(report, "scope_counts", scope_counts) + object.__setattr__(report, "port_bucket_counts", port_bucket_counts) + object.__setattr__(report, "duplicate_groups", duplicate_groups) + return report + + +def _create_privacy_receipt(report: PrivacyReport) -> PrivacyReceipt: + if type(report) is not PrivacyReport: + raise TypeError("report must come from build_privacy_report") + + report_sha256 = hashlib.sha256(report.canonical_json_bytes()).hexdigest() + receipt: PrivacyReceipt = object.__new__(PrivacyReceipt) + object.__setattr__(receipt, "report", report) + object.__setattr__(receipt, "report_sha256", report_sha256) + return receipt + + +def _port_bucket(port: int) -> str: + if port <= 1_023: + return "system_1_1023" + if port <= 49_151: + return "registered_1024_49151" + return "dynamic_49152_65535" + + +def _count_pairs(labels: tuple[str, ...], observed: Counter[str]) -> CountPairs: + return tuple((label, observed[label]) for label in labels) + + +def _report_from_corpus( + corpus: EndpointCorpus, + *, + pseudonymization_key: bytes, + payload: bytes, +) -> PrivacyReport: + version_counts: Counter[str] = Counter() + scope_counts: Counter[str] = Counter() + port_counts: Counter[str] = Counter() + canonical_counts: Counter[str] = Counter() + + for endpoint in corpus.endpoints: + version_counts["ipv6" if endpoint.version is IPVersion.IPV6 else "ipv4"] += 1 + scope_counts[endpoint.scope.value] += 1 + port_counts[_port_bucket(endpoint.port)] += 1 + canonical_counts[endpoint.canonical] += 1 + + duplicate_groups = tuple( + sorted( + ( + _create_duplicate_group( + group_id=_keyed_identifier( + pseudonymization_key, + domain=_DUPLICATE_GROUP_DOMAIN, + value=canonical.encode("ascii"), + prefix=_DUPLICATE_GROUP_ID_PREFIX, + ), + occurrences=occurrences, + ) + for canonical, occurrences in canonical_counts.items() + if occurrences > 1 + ), + key=lambda group: group.group_id, + ) + ) + return _create_privacy_report( + source_content_id=_keyed_identifier( + pseudonymization_key, + domain=_SOURCE_CONTENT_DOMAIN, + value=payload, + prefix=_SOURCE_CONTENT_ID_PREFIX, + ), + runtime_profile=_create_runtime_profile(), + source_bytes=corpus.source_bytes, + physical_line_count=corpus.physical_line_count, + endpoint_count=corpus.endpoint_count, + unique_endpoint_count=corpus.unique_count, + ip_version_counts=_count_pairs(_IP_VERSION_LABELS, version_counts), + scope_counts=_count_pairs(_SCOPE_LABELS, scope_counts), + port_bucket_counts=_count_pairs(_PORT_BUCKET_LABELS, port_counts), + duplicate_groups=duplicate_groups, + ) + + +def build_privacy_report( + payload: bytes, + *, + pseudonymization_key: bytes, +) -> PrivacyReport: + """Parse exact bytes locally and return only keyed IDs and aggregates.""" + + if type(payload) is not bytes: + raise TypeError("payload must be exact bytes") + _validate_pseudonymization_key(pseudonymization_key) + corpus = parse_corpus(payload) + return _report_from_corpus( + corpus, + pseudonymization_key=pseudonymization_key, + payload=payload, + ) + + +def build_privacy_receipt( + payload: bytes, + *, + pseudonymization_key: bytes, +) -> PrivacyReceipt: + """Build a public report and bind its canonical bytes in one pass.""" + + report = build_privacy_report( + payload, + pseudonymization_key=pseudonymization_key, + ) + return _create_privacy_receipt(report) diff --git a/src/netveil/py.typed b/src/netveil/py.typed new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/netveil/py.typed @@ -0,0 +1 @@ + diff --git a/src/netveil_bootstrap.py b/src/netveil_bootstrap.py new file mode 100644 index 0000000..e9f0b3b --- /dev/null +++ b/src/netveil_bootstrap.py @@ -0,0 +1,540 @@ +"""Stdlib-only integrity bootstrap for the installed Netveil command.""" + +from __future__ import annotations + +import base64 +import hashlib +import importlib +import importlib.abc +import importlib.util +import os +import stat +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from importlib import metadata +from pathlib import Path, PurePosixPath +from types import ModuleType +from typing import Final, NoReturn, TextIO + +_DISTRIBUTION_NAME: Final = "netveil-audit" +_DISTRIBUTION_VERSION: Final = "0.3.0" +_BOOTSTRAP_FILE: Final = "netveil_bootstrap.py" +_MAX_ARTIFACT_FILE_BYTES: Final = 1_048_576 +_ARTIFACT_FAILURE_EXIT: Final = 10 +_OUTPUT_FAILURE_EXIT: Final = 15 +_INTERNAL_FAILURE_EXIT: Final = 70 +_SOURCE_MODULES: Final = ( + ("netveil", "__init__.py", True), + ("netveil.cli", "cli.py", False), + ("netveil.model", "model.py", False), + ("netveil.parser", "parser.py", False), + ("netveil.privacy", "privacy.py", False), +) +_PACKAGE_FILES: Final = frozenset( + {filename for _, filename, _ in _SOURCE_MODULES} | {"py.typed"} +) +_METADATA_FILES: Final = ( + "METADATA", + "WHEEL", + "top_level.txt", +) + + +class _BootstrapFailure(Exception): + def __init__(self, code: str, exit_code: int) -> None: + super().__init__(code) + self.code = code + self.exit_code = exit_code + + +@dataclass(frozen=True, slots=True) +class _PinnedBytes: + payload: bytes + identity: tuple[int, ...] + + +@dataclass(frozen=True, slots=True) +class _VerifiedArtifact: + version: str + package_root: Path + sources: tuple[tuple[str, str, bytes, bool], ...] + + +def _fail(code: str = "artifact_unverified") -> NoReturn: + raise _BootstrapFailure(code, _ARTIFACT_FAILURE_EXIT) + + +def _identity(status: os.stat_result) -> tuple[int, ...]: + return ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_uid, + status.st_gid, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ) + + +def _file_flags() -> int: + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if os.name != "posix" or any(not hasattr(os, name) for name in required): + _fail("platform_unsupported") + return os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + + +def _directory_flags() -> int: + if not hasattr(os, "O_DIRECTORY"): + _fail("platform_unsupported") + return _file_flags() | os.O_DIRECTORY + + +def _close(descriptor: int) -> None: + try: + os.close(descriptor) + except OSError: + _fail() + + +def _read_descriptor( + descriptor: int, + *, + expected_size: int, +) -> _PinnedBytes: + if ( + type(expected_size) is not int + or expected_size < 0 + or expected_size > _MAX_ARTIFACT_FILE_BYTES + ): + _fail() + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_size != expected_size: + _fail() + chunks: list[bytes] = [] + observed = 0 + while observed <= expected_size: + chunk = os.read( + descriptor, + min(65_536, expected_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail() + if ( + len(payload) != expected_size + or _identity(before) != _identity(after) + or after.st_size != len(payload) + ): + _fail() + return _PinnedBytes(payload=payload, identity=_identity(after)) + + +def _record_contract(record: metadata.PackagePath) -> tuple[int, str]: + file_hash = record.hash + file_size = record.size + if ( + file_hash is None + or file_hash.mode != "sha256" + or type(file_size) is not int + or file_size < 0 + or file_size > _MAX_ARTIFACT_FILE_BYTES + ): + _fail() + return file_size, file_hash.value + + +def _sha256_record_value(payload: bytes) -> str: + digest = hashlib.sha256(payload).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def _read_path( + path: Path, + *, + record: metadata.PackagePath, +) -> _PinnedBytes: + expected_size, expected_hash = _record_contract(record) + descriptor = -1 + try: + descriptor = os.open(path, _file_flags()) + pinned = _read_descriptor(descriptor, expected_size=expected_size) + except OSError: + _fail() + finally: + if descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + _close(descriptor) + if _sha256_record_value(pinned.payload) != expected_hash: + _fail() + return pinned + + +def _read_package_file( + root_descriptor: int, + *, + name: str, + record: metadata.PackagePath, + distribution: metadata.Distribution, +) -> bytes: + expected_size, expected_hash = _record_contract(record) + descriptor = -1 + located_descriptor = -1 + try: + descriptor = os.open( + name, + _file_flags(), + dir_fd=root_descriptor, + ) + pinned = _read_descriptor(descriptor, expected_size=expected_size) + located = Path(str(distribution.locate_file(record))) + located_descriptor = os.open(located, _file_flags()) + located_status = os.fstat(located_descriptor) + current_status = os.stat( + name, + dir_fd=root_descriptor, + follow_symlinks=False, + ) + except OSError: + _fail() + finally: + if located_descriptor >= 0: + _close(located_descriptor) + if descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + _close(descriptor) + if ( + _identity(located_status) != pinned.identity + or _identity(current_status) != pinned.identity + or _sha256_record_value(pinned.payload) != expected_hash + ): + _fail() + return pinned.payload + + +def _cache_names() -> frozenset[str]: + cache_tag = sys.implementation.cache_tag + if type(cache_tag) is not str or not cache_tag: + _fail() + return frozenset( + f"{Path(filename).stem}.{cache_tag}.pyc" for _, filename, _ in _SOURCE_MODULES + ) + + +def _verify_cache_directory(root_descriptor: int) -> None: + descriptor = -1 + try: + descriptor = os.open( + "__pycache__", + _directory_flags(), + dir_fd=root_descriptor, + ) + before = os.fstat(descriptor) + if not stat.S_ISDIR(before.st_mode): + _fail() + names = os.listdir(descriptor) + if len(names) != len(set(names)) or not set(names).issubset(_cache_names()): + _fail() + for name in names: + status = os.stat(name, dir_fd=descriptor, follow_symlinks=False) + if not stat.S_ISREG(status.st_mode): + _fail() + after = os.fstat(descriptor) + except OSError: + _fail() + finally: + if descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + _close(descriptor) + if _identity(before) != _identity(after): + _fail() + + +def _scan_package_directory(root_descriptor: int) -> frozenset[str]: + try: + names = os.listdir(root_descriptor) + except OSError: + _fail() + if len(names) != len(set(names)): + _fail() + observed = frozenset(names) + allowed = _PACKAGE_FILES | {"__pycache__"} + if not _PACKAGE_FILES.issubset(observed) or not observed.issubset(allowed): + _fail() + if "__pycache__" in observed: + _verify_cache_directory(root_descriptor) + return observed + + +def _collect_records( + distribution: metadata.Distribution, +) -> tuple[ + dict[str, metadata.PackagePath], + metadata.PackagePath, + dict[str, metadata.PackagePath], +]: + files = distribution.files + if files is None: + _fail() + records: dict[str, metadata.PackagePath] = {} + for record in files: + raw_path = str(record) + if raw_path in records: + _fail() + records[raw_path] = record + + package_records: dict[str, metadata.PackagePath] = {} + allowed_cache_paths = {f"netveil/__pycache__/{name}" for name in _cache_names()} + for raw_path, record in records.items(): + path = PurePosixPath(raw_path) + if not path.parts or path.parts[0] != "netveil": + continue + if len(path.parts) == 2 and path.name in _PACKAGE_FILES: + if path.name in package_records: + _fail() + package_records[path.name] = record + elif raw_path not in allowed_cache_paths: + _fail() + if set(package_records) != _PACKAGE_FILES: + _fail() + + bootstrap_record = records.get(_BOOTSTRAP_FILE) + if bootstrap_record is None: + _fail() + + metadata_root = f"netveil_audit-{_DISTRIBUTION_VERSION}.dist-info" + metadata_records: dict[str, metadata.PackagePath] = {} + for name in _METADATA_FILES: + metadata_record = records.get(f"{metadata_root}/{name}") + if metadata_record is None: + _fail() + metadata_records[name] = metadata_record + return package_records, bootstrap_record, metadata_records + + +def _verify_distribution_metadata( + distribution: metadata.Distribution, + records: Mapping[str, metadata.PackagePath], +) -> None: + try: + project_name = distribution.metadata["Name"] + project_version = distribution.metadata["Version"] + except KeyError: + _fail() + if ( + distribution.version != _DISTRIBUTION_VERSION + or project_name != _DISTRIBUTION_NAME + or project_version != _DISTRIBUTION_VERSION + ): + _fail() + if list(distribution.entry_points): + _fail() + + payloads = { + name: _read_path( + Path(str(distribution.locate_file(record))), + record=record, + ).payload + for name, record in records.items() + } + if payloads["top_level.txt"] != b"netveil\nnetveil_bootstrap\n": + _fail() + + +def _verify_installed_artifact() -> _VerifiedArtifact: + """Bind installed source bytes before any ``netveil`` package import.""" + + if any(name == "netveil" or name.startswith("netveil.") for name in sys.modules): + _fail() + try: + distribution = metadata.distribution(_DISTRIBUTION_NAME) + except metadata.PackageNotFoundError: + _fail() + + package_records, bootstrap_record, metadata_records = _collect_records(distribution) + _verify_distribution_metadata(distribution, metadata_records) + + bootstrap_path = Path(__file__) + bootstrap_pinned = _read_path(bootstrap_path, record=bootstrap_record) + try: + recorded_bootstrap = Path(str(distribution.locate_file(bootstrap_record))) + recorded_pinned = _read_path( + recorded_bootstrap, + record=bootstrap_record, + ) + except OSError: + _fail() + if bootstrap_pinned.identity != recorded_pinned.identity: + _fail() + + package_root = bootstrap_path.parent / "netveil" + root_descriptor = -1 + recorded_root_descriptor = -1 + try: + root_descriptor = os.open(package_root, _directory_flags()) + root_before = os.fstat(root_descriptor) + if not stat.S_ISDIR(root_before.st_mode): + _fail() + recorded_root = Path(str(distribution.locate_file("netveil"))) + recorded_root_descriptor = os.open(recorded_root, _directory_flags()) + recorded_root_status = os.fstat(recorded_root_descriptor) + if _identity(root_before) != _identity(recorded_root_status): + _fail() + observed_before = _scan_package_directory(root_descriptor) + payloads = { + name: _read_package_file( + root_descriptor, + name=name, + record=package_records[name], + distribution=distribution, + ) + for name in sorted(_PACKAGE_FILES) + } + observed_after = _scan_package_directory(root_descriptor) + root_after = os.fstat(root_descriptor) + except OSError: + _fail() + finally: + if recorded_root_descriptor >= 0: + _close(recorded_root_descriptor) + if root_descriptor >= 0: # pragma: no branch - acquisition failure re-raises. + _close(root_descriptor) + if observed_before != observed_after or _identity(root_before) != _identity( + root_after + ): + _fail() + + sources = tuple( + (module, filename, payloads[filename], is_package) + for module, filename, is_package in _SOURCE_MODULES + ) + return _VerifiedArtifact( + version=distribution.version, + package_root=package_root, + sources=sources, + ) + + +class _VerifiedSourceLoader(importlib.abc.Loader): + def __init__(self, *, source: bytes, origin: str) -> None: + self._source = source + self._origin = origin + + def create_module(self, spec: importlib.machinery.ModuleSpec) -> ModuleType | None: + del spec + return None + + def exec_module(self, module: ModuleType) -> None: + code = compile( + self._source, + self._origin, + "exec", + dont_inherit=True, + ) + exec(code, module.__dict__) # noqa: S102 - executes only verified bytes. + + +class _VerifiedSourceFinder(importlib.abc.MetaPathFinder): + def __init__(self, artifact: _VerifiedArtifact) -> None: + self._sources = { + module: ( + artifact.package_root / filename, + payload, + is_package, + ) + for module, filename, payload, is_package in artifact.sources + } + + def find_spec( + self, + fullname: str, + path: Sequence[str] | None, + target: ModuleType | None = None, + ) -> importlib.machinery.ModuleSpec | None: + del path, target + if fullname != "netveil" and not fullname.startswith("netveil."): + return None + source = self._sources.get(fullname) + if source is None: + raise ModuleNotFoundError("verified Netveil module is not allowed") + origin, payload, is_package = source + loader = _VerifiedSourceLoader(source=payload, origin=str(origin)) + spec = importlib.util.spec_from_loader( + fullname, + loader, + origin=str(origin), + is_package=is_package, + ) + if spec is None: + raise ModuleNotFoundError("verified Netveil module is unavailable") + return spec + + +def _load_verified_cli(artifact: _VerifiedArtifact) -> ModuleType: + if any(name == "netveil" or name.startswith("netveil.") for name in sys.modules): + _fail() + finder = _VerifiedSourceFinder(artifact) + sys.meta_path.insert(0, finder) + try: + cli = importlib.import_module("netveil.cli") + except BaseException: # Cleanup must cover import-time exits. + for name in tuple(sys.modules): + if name == "netveil" or name.startswith("netveil."): + sys.modules.pop(name, None) + if finder in sys.meta_path: + sys.meta_path.remove(finder) + raise + return cli + + +def _write_text(stream: TextIO, payload: str) -> bool: + offset = 0 + try: + while offset < len(payload): + written = stream.write(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return False + offset += written + stream.flush() + except OSError: + return False + return True + + +def _emit_failure(code: str, exit_code: int) -> int: + if not _write_text(sys.stderr, f"netveil-audit: {code}\n"): + return _OUTPUT_FAILURE_EXIT + return exit_code + + +def main(argv: Sequence[str] | None = None) -> int: + """Verify, source-load, and execute the installed command.""" + + try: + artifact = _verify_installed_artifact() + cli = _load_verified_cli(artifact) + run = cli._main_verified + result = run( + argv, + verified_distribution_version=artifact.version, + ) + if type(result) is not int: + raise TypeError + return result + except _BootstrapFailure as failure: + return _emit_failure(failure.code, failure.exit_code) + except KeyboardInterrupt: + return _emit_failure("interrupted", _INTERNAL_FAILURE_EXIT) + except SystemExit: + return _emit_failure("internal_error", _INTERNAL_FAILURE_EXIT) + except Exception: # noqa: BLE001 - never expose bootstrap tracebacks. + return _emit_failure("internal_error", _INTERNAL_FAILURE_EXIT) diff --git a/tests/fixtures/launcher_bootstrap_fixture.py b/tests/fixtures/launcher_bootstrap_fixture.py new file mode 100644 index 0000000..21e1dad --- /dev/null +++ b/tests/fixtures/launcher_bootstrap_fixture.py @@ -0,0 +1,7 @@ +"""Minimal source-loaded bootstrap fixture for launcher boundary tests.""" + + +def main(arguments: list[str]) -> int: + if arguments[0] == "explode": + raise RuntimeError("fixture failure") + return {"invalid": True, "safe": 23}[arguments[0]] diff --git a/tests/test_bootstrap.py b/tests/test_bootstrap.py new file mode 100644 index 0000000..d7fa81a --- /dev/null +++ b/tests/test_bootstrap.py @@ -0,0 +1,1040 @@ +from __future__ import annotations + +import base64 +import contextlib +import hashlib +import importlib +import io +import os +import py_compile +import stat +import subprocess +import sys +import tempfile +import unittest +from collections.abc import Callable, Iterator +from dataclasses import dataclass, replace +from importlib import metadata +from pathlib import Path +from types import ModuleType +from typing import cast +from unittest.mock import patch + +import netveil_bootstrap as bootstrap + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_ROOT = ROOT / "src" + + +@dataclass(slots=True) +class _FakeHash: + mode: str + value: str + + +@dataclass(slots=True) +class _FakeRecord: + path: str + hash: _FakeHash | None + size: int | None + + def __str__(self) -> str: + return self.path + + +@dataclass(slots=True) +class _FakeEntryPoint: + group: str + name: str + value: str + + +@dataclass +class _FakeDistribution: + root: Path + version: str + files: list[_FakeRecord] | None + metadata: dict[str, str] + entry_points: list[_FakeEntryPoint] + + def locate_file(self, path: object) -> Path: + return self.root / str(path) + + +def _hash(payload: bytes) -> str: + digest = hashlib.sha256(payload).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def _record(path: str, payload: bytes) -> _FakeRecord: + return _FakeRecord(path, _FakeHash("sha256", _hash(payload)), len(payload)) + + +def _write(root: Path, relative: str, payload: bytes) -> _FakeRecord: + target = root / relative + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(payload) + return _record(relative, payload) + + +def _create_artifact(root: Path) -> _FakeDistribution: + records: list[_FakeRecord] = [] + records.append( + _write( + root, + bootstrap._BOOTSTRAP_FILE, + (SOURCE_ROOT / bootstrap._BOOTSTRAP_FILE).read_bytes(), + ) + ) + for name in sorted(bootstrap._PACKAGE_FILES): + records.append( + _write( + root, + f"netveil/{name}", + (SOURCE_ROOT / "netveil" / name).read_bytes(), + ) + ) + + metadata_root = f"netveil_audit-{bootstrap._DISTRIBUTION_VERSION}.dist-info" + metadata_payloads = { + "METADATA": (b"Metadata-Version: 2.4\nName: netveil-audit\nVersion: 0.3.0\n"), + "WHEEL": b"Wheel-Version: 1.0\nRoot-Is-Purelib: true\nTag: py3-none-any\n", + "top_level.txt": b"netveil\nnetveil_bootstrap\n", + } + for name, payload in metadata_payloads.items(): + records.append(_write(root, f"{metadata_root}/{name}", payload)) + + return _FakeDistribution( + root=root, + version=bootstrap._DISTRIBUTION_VERSION, + files=records, + metadata={ + "Name": bootstrap._DISTRIBUTION_NAME, + "Version": bootstrap._DISTRIBUTION_VERSION, + }, + entry_points=[], + ) + + +@contextlib.contextmanager +def _isolated_netveil_modules() -> Iterator[None]: + saved_modules = { + name: module + for name, module in sys.modules.items() + if name == "netveil" or name.startswith("netveil.") + } + saved_meta_path = list(sys.meta_path) + for name in saved_modules: + sys.modules.pop(name, None) + try: + yield + finally: + for name in tuple(sys.modules): + if name == "netveil" or name.startswith("netveil."): + sys.modules.pop(name, None) + sys.modules.update(saved_modules) + sys.meta_path[:] = saved_meta_path + + +def _verify( + root: Path, + distribution: _FakeDistribution, +) -> bootstrap._VerifiedArtifact: + with ( + _isolated_netveil_modules(), + patch.object(bootstrap, "__file__", str(root / bootstrap._BOOTSTRAP_FILE)), + patch.object( + metadata, + "distribution", + return_value=cast(metadata.Distribution, distribution), + ), + ): + return bootstrap._verify_installed_artifact() + + +class BootstrapArtifactTests(unittest.TestCase): + def test_importing_bootstrap_does_not_import_package(self) -> None: + command = ( + "import sys;" + f"sys.path.insert(0,{str(SOURCE_ROOT)!r});" + "import netveil_bootstrap;" + "print(any(n == 'netveil' or n.startswith('netveil.') " + "for n in sys.modules))" + ) + completed = subprocess.run( + (sys.executable, "-I", "-B", "-c", command), + check=True, + capture_output=True, + text=True, + env={ + "HOME": "/nonexistent", + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": os.environ.get("PATH", ""), + "PYTHONHASHSEED": "0", + "TZ": "UTC", + }, + ) + self.assertEqual(completed.stdout, "False\n") + self.assertEqual(completed.stderr, "") + + def test_exact_artifact_returns_pinned_source_bytes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + distribution = _create_artifact(root) + artifact = _verify(root, distribution) + self.assertEqual(artifact.version, bootstrap._DISTRIBUTION_VERSION) + self.assertEqual( + {module for module, _, _, _ in artifact.sources}, + {module for module, _, _ in bootstrap._SOURCE_MODULES}, + ) + self.assertTrue(all(payload for _, _, payload, _ in artifact.sources)) + + def test_loaded_package_and_missing_distribution_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + _create_artifact(root) + with ( + patch.dict(sys.modules, {"netveil": ModuleType("netveil")}), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_installed_artifact() + + with ( + _isolated_netveil_modules(), + patch.object( + metadata, + "distribution", + side_effect=metadata.PackageNotFoundError("netveil-audit"), + ), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_installed_artifact() + + def test_metadata_identity_and_entrypoint_are_exact(self) -> None: + mutations = ("version", "name", "metadata-version", "entrypoint") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + if mutation == "version": + distribution.version = "9.9.9" + elif mutation == "name": + distribution.metadata["Name"] = "other" + elif mutation == "metadata-version": + distribution.metadata["Version"] = "9.9.9" + else: + distribution.entry_points.append( + _FakeEntryPoint( + group="console_scripts", + name="netveil-audit", + value="netveil.cli:entrypoint", + ) + ) + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_missing_duplicate_nested_and_extra_package_records_fail(self) -> None: + mutations = ("missing", "duplicate", "nested", "extra-disk") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + assert distribution.files is not None + if mutation == "missing": + distribution.files = [ + record + for record in distribution.files + if str(record) != "netveil/parser.py" + ] + elif mutation == "duplicate": + distribution.files.append(distribution.files[0]) + elif mutation == "nested": + distribution.files.append( + _FakeRecord( + "netveil/plugins/evil.py", + _FakeHash("sha256", "x"), + 1, + ) + ) + else: + (root / "netveil" / "evil.py").write_text("raise SystemExit\n") + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_hash_size_type_and_location_drift_fail(self) -> None: + mutations = ( + "hash", + "algorithm", + "size", + "size-type", + "oversized", + "symlink", + ) + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + assert distribution.files is not None + record = next( + item + for item in distribution.files + if str(item) == "netveil/parser.py" + ) + if mutation == "hash": + assert record.hash is not None + record.hash.value = "wrong" + elif mutation == "algorithm": + assert record.hash is not None + record.hash.mode = "sha512" + elif mutation == "size": + assert record.size is not None + record.size += 1 + elif mutation == "size-type": + record.size = True + elif mutation == "oversized": + record.size = bootstrap._MAX_ARTIFACT_FILE_BYTES + 1 + else: + target = root / "netveil" / "parser.py" + target.unlink() + target.symlink_to(root / "netveil" / "model.py") + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_metadata_payloads_and_top_levels_are_record_bound(self) -> None: + mutations = ("payload", "top-level", "entry-point", "missing-record") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + assert distribution.files is not None + metadata_root = ( + f"netveil_audit-{bootstrap._DISTRIBUTION_VERSION}.dist-info" + ) + if mutation == "payload": + (root / metadata_root / "WHEEL").write_bytes(b"tampered\n") + elif mutation == "top-level": + path = root / metadata_root / "top_level.txt" + payload = b"netveil_bootstrap\nnetveil\n" + path.write_bytes(payload) + record = next( + item + for item in distribution.files + if str(item).endswith("/top_level.txt") + ) + record.hash = _FakeHash("sha256", _hash(payload)) + record.size = len(payload) + elif mutation == "entry-point": + distribution.entry_points.append( + _FakeEntryPoint( + group="console_scripts", + name="other", + value="other:main", + ) + ) + else: + distribution.files = [ + item + for item in distribution.files + if not str(item).endswith("/METADATA") + ] + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_cache_is_inert_but_its_shape_is_bounded(self) -> None: + mutations = ("valid", "unknown", "symlink", "directory") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + package_root = root / "netveil" + cache = package_root / "__pycache__" + cache.mkdir() + known = next(iter(sorted(bootstrap._cache_names()))) + target = cache / known + if mutation == "valid": + target.write_bytes(b"untrusted and deliberately unread") + assert distribution.files is not None + distribution.files.append( + _FakeRecord( + f"netveil/__pycache__/{known}", + None, + None, + ) + ) + artifact = _verify(root, distribution) + self.assertEqual( + artifact.version, + bootstrap._DISTRIBUTION_VERSION, + ) + continue + if mutation == "unknown": + target = cache / "evil.pyc" + target.write_bytes(b"x") + elif mutation == "symlink": + target.symlink_to(package_root / "model.py") + else: + target.mkdir() + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + +class BootstrapSourceLoaderTests(unittest.TestCase): + def test_verified_bytes_ignore_disk_tamper_and_unchecked_hash_pyc(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + distribution = _create_artifact(root) + package_root = root / "netveil" + cache = package_root / "__pycache__" + cache.mkdir() + cache_name = f"cli.{sys.implementation.cache_tag}.pyc" + malicious_source = root / "malicious.py" + marker = root / "BYTECODE_EXECUTED" + malicious_source.write_text( + "from pathlib import Path\n" + f"Path({str(marker)!r}).write_text('executed')\n" + ) + py_compile.compile( + str(malicious_source), + cfile=str(cache / cache_name), + doraise=True, + invalidation_mode=py_compile.PycInvalidationMode.UNCHECKED_HASH, + ) + assert distribution.files is not None + distribution.files.append( + _FakeRecord( + f"netveil/__pycache__/{cache_name}", + None, + None, + ) + ) + + with ( + _isolated_netveil_modules(), + patch.object( + bootstrap, + "__file__", + str(root / bootstrap._BOOTSTRAP_FILE), + ), + patch.object( + metadata, + "distribution", + return_value=cast(metadata.Distribution, distribution), + ), + ): + artifact = bootstrap._verify_installed_artifact() + artifact = replace( + artifact, + package_root=SOURCE_ROOT / "netveil", + ) + (package_root / "cli.py").write_text( + "raise AssertionError('disk source executed')\n" + ) + cli = bootstrap._load_verified_cli(artifact) + self.assertEqual( + cli._DISTRIBUTION_VERSION, + bootstrap._DISTRIBUTION_VERSION, + ) + with self.assertRaises(ModuleNotFoundError): + importlib.import_module("netveil.evil") + self.assertFalse(marker.exists()) + + def test_finder_ignores_other_modules_and_rejects_unknown_netveil_module( + self, + ) -> None: + artifact = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=Path("/verified"), + sources=(("netveil", "__init__.py", b"", True),), + ) + finder = bootstrap._VerifiedSourceFinder(artifact) + self.assertIsNone(finder.find_spec("json", None)) + with self.assertRaises(ModuleNotFoundError): + finder.find_spec("netveil.evil", None) + spec = finder.find_spec("netveil", None) + self.assertIsNotNone(spec) + + +class BootstrapBoundaryTests(unittest.TestCase): + def test_descriptor_reader_rejects_growth_nonregular_and_read_failure( + self, + ) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "payload" + path.write_bytes(b"abc") + descriptor = os.open(path, os.O_RDONLY) + try: + current = os.fstat(descriptor) + changed = os.stat_result( + ( + current.st_mode, + current.st_ino, + current.st_dev, + current.st_nlink, + current.st_uid, + current.st_gid, + current.st_size + 1, + current.st_atime, + current.st_mtime, + current.st_ctime, + ) + ) + with ( + patch.object(os, "fstat", side_effect=(current, changed)), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_descriptor(descriptor, expected_size=3) + with ( + patch.object(os, "read", side_effect=OSError("PRIVATE")), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_descriptor(descriptor, expected_size=3) + finally: + os.close(descriptor) + + descriptor, writable = os.pipe() + try: + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._read_descriptor(descriptor, expected_size=0) + finally: + os.close(descriptor) + os.close(writable) + + def test_platform_and_record_contract_fail_closed(self) -> None: + with ( + patch.object(os, "name", "unsupported"), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._file_flags() + record = cast( + metadata.PackagePath, + _FakeRecord("x", None, None), + ) + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._record_contract(record) + + def test_directory_flag_close_and_invalid_size_failures(self) -> None: + directory_flag = os.O_DIRECTORY + del os.O_DIRECTORY + try: + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._directory_flags() + finally: + os.O_DIRECTORY = directory_flag # type: ignore[misc] + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "payload" + path.write_bytes(b"x") + descriptor = os.open(path, os.O_RDONLY) + try: + with ( + patch.object(os, "close", side_effect=OSError("PRIVATE")), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._close(descriptor) + finally: + os.close(descriptor) + + for invalid in (-1, True, bootstrap._MAX_ARTIFACT_FILE_BYTES + 1): + with ( + self.subTest(invalid=invalid), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_descriptor(-1, expected_size=invalid) + + def test_path_reader_rejects_missing_same_size_tamper_and_overread(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + payload = b"abc" + path = root / "payload" + path.write_bytes(payload) + record = cast(metadata.PackagePath, _record("payload", payload)) + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._read_path(root / "missing", record=record) + with ( + patch.object(os, "open", return_value=-1), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_path(path, record=record) + + wrong = cast( + metadata.PackagePath, + _FakeRecord( + "payload", + _FakeHash("sha256", _hash(b"xyz")), + len(payload), + ), + ) + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._read_path(path, record=wrong) + + descriptor = os.open(path, os.O_RDONLY) + try: + current = os.fstat(descriptor) + with ( + patch.object(os, "fstat", side_effect=(current, current)), + patch.object(os, "read", return_value=b"abcd"), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._read_descriptor(descriptor, expected_size=3) + finally: + os.close(descriptor) + + def test_cache_and_package_scan_low_level_failures(self) -> None: + implementation = sys.implementation + with ( + patch.object( + sys, + "implementation", + type("Implementation", (), {"cache_tag": None})(), + ), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._cache_names() + self.assertIs(sys.implementation, implementation) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cache = root / "__pycache__" + cache.mkdir() + root_descriptor = os.open(root, os.O_RDONLY | os.O_DIRECTORY) + cache_descriptor = os.open(cache, os.O_RDONLY | os.O_DIRECTORY) + try: + cache_status = os.fstat(cache_descriptor) + regular_status = os.stat_result( + ( + stat.S_IFREG | 0o600, + cache_status.st_ino, + cache_status.st_dev, + cache_status.st_nlink, + cache_status.st_uid, + cache_status.st_gid, + cache_status.st_size, + cache_status.st_atime, + cache_status.st_mtime, + cache_status.st_ctime, + ) + ) + with ( + patch.object(os, "open", return_value=cache_descriptor), + patch.object(os, "fstat", return_value=regular_status), + patch.object(os, "close"), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_cache_directory(root_descriptor) + + with ( + patch.object(os, "open", side_effect=OSError("PRIVATE")), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_cache_directory(root_descriptor) + with ( + patch.object(os, "open", return_value=-1), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_cache_directory(root_descriptor) + + changed_status = os.stat_result( + ( + cache_status.st_mode, + cache_status.st_ino, + cache_status.st_dev, + cache_status.st_nlink, + cache_status.st_uid, + cache_status.st_gid, + cache_status.st_size, + cache_status.st_atime, + cache_status.st_mtime + 1, + cache_status.st_ctime, + ) + ) + with ( + patch.object(os, "open", return_value=cache_descriptor), + patch.object( + os, + "fstat", + side_effect=(cache_status, changed_status), + ), + patch.object(os, "listdir", return_value=[]), + patch.object(os, "close"), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._verify_cache_directory(root_descriptor) + + with ( + patch.object(os, "listdir", side_effect=OSError("PRIVATE")), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._scan_package_directory(root_descriptor) + with ( + patch.object(os, "listdir", return_value=["x", "x"]), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._scan_package_directory(root_descriptor) + finally: + os.close(cache_descriptor) + os.close(root_descriptor) + + def test_record_collection_and_metadata_rare_failures(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + distribution = _create_artifact(root) + distribution.files = None + with self.assertRaises(bootstrap._BootstrapFailure): + bootstrap._collect_records(cast(metadata.Distribution, distribution)) + + mutations = ("normalized-duplicate", "bootstrap", "metadata-key", "unicode") + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + assert distribution.files is not None + if mutation == "normalized-duplicate": + source = next( + item + for item in distribution.files + if str(item) == "netveil/parser.py" + ) + distribution.files.append( + _FakeRecord( + "netveil/./parser.py", + source.hash, + source.size, + ) + ) + elif mutation == "bootstrap": + distribution.files = [ + item + for item in distribution.files + if str(item) != bootstrap._BOOTSTRAP_FILE + ] + elif mutation == "metadata-key": + distribution.metadata.pop("Name") + else: + metadata_root = ( + f"netveil_audit-{bootstrap._DISTRIBUTION_VERSION}.dist-info" + ) + path = root / metadata_root / "top_level.txt" + payload = b"\xffetveil\nnetveil_bootstrap\n" + path.write_bytes(payload) + record = next( + item + for item in distribution.files + if str(item).endswith("/top_level.txt") + ) + record.hash = _FakeHash("sha256", _hash(payload)) + record.size = len(payload) + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + + def test_artifact_location_directory_and_terminal_identity_failures(self) -> None: + mutations = ( + "bootstrap-locate", + "bootstrap-inode", + "root-kind", + "root-locate", + "root-inode", + "root-missing", + "terminal-drift", + ) + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + root = Path(directory) + distribution = _create_artifact(root) + original_locate = distribution.locate_file + locate_override: Callable[[object], Path] | None = None + if mutation == "bootstrap-locate": + + def locate_bootstrap_failure( + path: object, + fallback: Callable[[object], Path] = original_locate, + ) -> Path: + if str(path) == bootstrap._BOOTSTRAP_FILE: + raise OSError("PRIVATE") + return fallback(path) + + locate_override = locate_bootstrap_failure + elif mutation == "bootstrap-inode": + copy = root / "bootstrap-copy.py" + copy.write_bytes((root / bootstrap._BOOTSTRAP_FILE).read_bytes()) + + def locate_bootstrap_copy( + path: object, + copy_path: Path = copy, + fallback: Callable[[object], Path] = original_locate, + ) -> Path: + if str(path) == bootstrap._BOOTSTRAP_FILE: + return copy_path + return fallback(path) + + locate_override = locate_bootstrap_copy + elif mutation == "root-kind": + kind = stat.S_ISDIR + package_mode = (root / "netveil").stat().st_mode + + def is_directory( + mode: int, + *, + expected_mode: int = package_mode, + fallback: Callable[[int], bool] = kind, + ) -> bool: + if mode == expected_mode: + return False + return fallback(mode) + + with ( + patch.object(stat, "S_ISDIR", side_effect=is_directory), + self.assertRaises(bootstrap._BootstrapFailure), + ): + _verify(root, distribution) + continue + elif mutation == "root-locate": + + def locate_root_failure( + path: object, + fallback: Callable[[object], Path] = original_locate, + ) -> Path: + if str(path) == "netveil": + raise OSError("PRIVATE") + return fallback(path) + + locate_override = locate_root_failure + elif mutation == "root-inode": + other = root / "other-netveil" + other.mkdir() + + def locate_other_root( + path: object, + other_root: Path = other, + fallback: Callable[[object], Path] = original_locate, + ) -> Path: + if str(path) == "netveil": + return other_root + return fallback(path) + + locate_override = locate_other_root + elif mutation == "root-missing": + package_root = root / "netveil" + moved_root = root / "moved-netveil" + package_root.rename(moved_root) + with self.assertRaises(bootstrap._BootstrapFailure): + _verify(root, distribution) + continue + else: + original_scan = bootstrap._scan_package_directory + calls = 0 + + def drift( + descriptor: int, + *, + scan: Callable[[int], frozenset[str]] = original_scan, + package_root: Path = root / "netveil", + ) -> frozenset[str]: + nonlocal calls + calls += 1 + observed = scan(descriptor) + if calls == 2: + os.utime(package_root, None) + return observed + + with ( + patch.object( + bootstrap, + "_scan_package_directory", + side_effect=drift, + ), + self.assertRaises(bootstrap._BootstrapFailure), + ): + _verify(root, distribution) + continue + self.assertIsNotNone(locate_override) + with ( + patch.object( + distribution, + "locate_file", + side_effect=locate_override, + ), + self.assertRaises(bootstrap._BootstrapFailure), + ): + _verify(root, distribution) + + def test_loader_preload_spec_and_import_failure_cleanup(self) -> None: + artifact = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=Path("/verified"), + sources=(("netveil", "__init__.py", b"", True),), + ) + with ( + patch.dict(sys.modules, {"netveil": ModuleType("netveil")}), + self.assertRaises(bootstrap._BootstrapFailure), + ): + bootstrap._load_verified_cli(artifact) + + finder = bootstrap._VerifiedSourceFinder(artifact) + with ( + patch.object(importlib.util, "spec_from_loader", return_value=None), + self.assertRaises(ModuleNotFoundError), + ): + finder.find_spec("netveil", None) + + broken = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=SOURCE_ROOT / "netveil", + sources=( + ( + "netveil", + "__init__.py", + ( + b"import sys, types\n" + b"sys.modules['netveil.partial'] = " + b"types.ModuleType('netveil.partial')\n" + b"raise RuntimeError('broken')\n" + ), + True, + ), + ), + ) + with _isolated_netveil_modules(): + before = list(sys.meta_path) + with self.assertRaises(RuntimeError): + bootstrap._load_verified_cli(broken) + self.assertEqual(sys.meta_path, before) + self.assertNotIn("netveil", sys.modules) + self.assertNotIn("netveil.partial", sys.modules) + + removes_finder = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=SOURCE_ROOT / "netveil", + sources=( + ( + "netveil", + "__init__.py", + b"import sys\nsys.meta_path.pop(0)\nraise RuntimeError('broken')\n", + True, + ), + ), + ) + with _isolated_netveil_modules(): + before = list(sys.meta_path) + with self.assertRaises(RuntimeError): + bootstrap._load_verified_cli(removes_finder) + self.assertEqual(sys.meta_path, before) + + def test_text_writer_oserror_and_successful_main_result(self) -> None: + stream = io.StringIO() + with patch.object(stream, "write", side_effect=OSError("PRIVATE")): + self.assertFalse(bootstrap._write_text(stream, "x")) + + artifact = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=Path("/verified"), + sources=(), + ) + cli = ModuleType("netveil.cli") + + def run( + argv: object, + *, + verified_distribution_version: str, + ) -> int: + self.assertEqual(argv, ["--version"]) + self.assertEqual(verified_distribution_version, "0.3.0") + return 23 + + cli._main_verified = run # type: ignore[attr-defined] + with ( + patch.object( + bootstrap, + "_verify_installed_artifact", + return_value=artifact, + ), + patch.object(bootstrap, "_load_verified_cli", return_value=cli), + ): + self.assertEqual(bootstrap.main(["--version"]), 23) + + def test_bootstrap_main_redacts_failures_and_validates_result_type(self) -> None: + stderr = io.StringIO() + cases = ( + ( + bootstrap._BootstrapFailure("artifact_unverified", 10), + 10, + "artifact_unverified", + ), + (KeyboardInterrupt(), 70, "interrupted"), + (SystemExit(99), 70, "internal_error"), + (RuntimeError("PRIVATE"), 70, "internal_error"), + ) + for failure, expected_exit, expected_code in cases: + with ( + self.subTest(expected_code=expected_code), + patch.object( + bootstrap, + "_verify_installed_artifact", + side_effect=failure, + ), + patch.object(sys, "stderr", stderr), + ): + self.assertEqual(bootstrap.main([]), expected_exit) + self.assertEqual( + stderr.getvalue(), + f"netveil-audit: {expected_code}\n", + ) + stderr.seek(0) + stderr.truncate() + + artifact = bootstrap._VerifiedArtifact( + version="0.3.0", + package_root=Path("/verified"), + sources=(), + ) + cli = ModuleType("netveil.cli") + cli._main_verified = ( # type: ignore[attr-defined] + lambda *args, **kwargs: "not-an-int" + ) + with ( + patch.object( + bootstrap, + "_verify_installed_artifact", + return_value=artifact, + ), + patch.object(bootstrap, "_load_verified_cli", return_value=cli), + patch.object(sys, "stderr", stderr), + ): + self.assertEqual(bootstrap.main([]), bootstrap._INTERNAL_FAILURE_EXIT) + self.assertEqual(stderr.getvalue(), "netveil-audit: internal_error\n") + + def test_short_stderr_write_is_bounded(self) -> None: + stderr = io.StringIO() + with ( + patch.object(stderr, "write", return_value=0), + patch.object(sys, "stderr", stderr), + ): + self.assertEqual( + bootstrap._emit_failure("safe", 10), + bootstrap._OUTPUT_FAILURE_EXIT, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..022a639 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,696 @@ +from __future__ import annotations + +import hashlib +import io +import json +import os +import socket +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import TextIO, cast +from unittest.mock import patch + +from netveil import ( + MIN_PSEUDONYMIZATION_KEY_BYTES, + build_privacy_receipt, + cli, +) +from netveil.cli import CliExitCode + +_DEMO_KEY = bytes(range(MIN_PSEUDONYMIZATION_KEY_BYTES)) +_DEMO_CORPUS = ( + b"# IETF documentation ranges only\n" + b"192.0.2.10:443\n" + b"192.0.2.10:443\n" + b"[2001:db8::10]:8443\n" +) + + +class _CapturedStdout: + def __init__(self) -> None: + self.buffer = io.BytesIO() + self.text = io.StringIO() + + def write(self, value: str) -> int: + return self.text.write(value) + + def flush(self) -> None: + self.text.flush() + self.buffer.flush() + + +def _write(path: Path, payload: bytes, mode: int = 0o600) -> None: + path.write_bytes(payload) + path.chmod(mode) + + +class CliWorkflowTests(unittest.TestCase): + def _run_main(self, arguments: list[str] | None) -> tuple[int, bytes, str, str]: + stdout = _CapturedStdout() + stderr = io.StringIO() + with ( + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified( + arguments, + verified_distribution_version=cli._DISTRIBUTION_VERSION, + ) + return ( + result, + stdout.buffer.getvalue(), + stdout.text.getvalue(), + stderr.getvalue(), + ) + + def test_receipt_command_emits_exact_public_bytes_without_network(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "private-corpus.txt" + key_path = root / "private-key.bin" + _write(corpus_path, _DEMO_CORPUS) + _write(key_path, _DEMO_KEY) + + with ( + patch.object( + socket, + "socket", + side_effect=AssertionError("network"), + ), + patch.object( + socket, + "getaddrinfo", + side_effect=AssertionError("resolution"), + ), + patch.object( + subprocess, + "Popen", + side_effect=AssertionError("process"), + ), + patch.object( + subprocess, + "run", + side_effect=AssertionError("process"), + ), + ): + result, binary_output, text_output, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(key_path), + ] + ) + + expected = build_privacy_receipt( + _DEMO_CORPUS, + pseudonymization_key=_DEMO_KEY, + ).canonical_json_bytes() + self.assertEqual(result, CliExitCode.SUCCESS) + self.assertEqual(binary_output, expected + b"\n") + self.assertEqual(text_output, "") + self.assertEqual(error_output, "") + document = json.loads(binary_output) + self.assertEqual(document["schema"], "netveil.aggregate-receipt.v1") + self.assertNotIn("192.0.2.10", binary_output.decode()) + self.assertNotIn(_DEMO_KEY.hex(), binary_output.decode()) + self.assertNotIn( + hashlib.sha256(_DEMO_CORPUS).hexdigest(), binary_output.decode() + ) + + def test_version_is_bound_to_verified_distribution(self) -> None: + result, binary_output, text_output, error_output = self._run_main(["--version"]) + self.assertEqual(result, CliExitCode.SUCCESS) + self.assertEqual(binary_output, b"") + self.assertEqual(text_output, "netveil-audit 0.3.0\n") + self.assertEqual(error_output, "") + with patch.object(sys, "argv", ["netveil-audit", "--version"]): + result, binary_output, text_output, error_output = self._run_main(None) + self.assertEqual(result, CliExitCode.SUCCESS) + self.assertEqual(binary_output, b"") + self.assertEqual(text_output, "netveil-audit 0.3.0\n") + self.assertEqual(error_output, "") + + def test_version_cannot_bypass_or_mix_with_a_workflow(self) -> None: + marker = "PRIVATE-VERSION-MIX-MARKER" + for arguments in ( + ["--version", "--version"], + ["--version", f"--unknown={marker}"], + [ + "--version", + "receipt", + f"{marker}-corpus", + "--key-file", + f"{marker}-key", + ], + ): + with self.subTest(arguments=arguments): + result, binary, text, error = self._run_main(arguments) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(binary, b"") + self.assertEqual(text, "") + self.assertEqual(error, "netveil-audit: usage_error\n") + self.assertNotIn(marker, error) + + def test_key_option_must_appear_exactly_once(self) -> None: + marker = "PRIVATE-REPEATED-KEY-MARKER" + for arguments in ( + [ + "receipt", + f"{marker}-corpus", + "--key-file", + f"{marker}-first", + "--key-file", + f"{marker}-second", + ], + [ + "receipt", + f"{marker}-corpus", + f"--key-file={marker}-first", + "--key-file", + f"{marker}-second", + ], + ["receipt", f"{marker}-corpus", "--key-file="], + ): + with self.subTest(arguments=arguments): + result, binary, text, error = self._run_main(arguments) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(binary, b"") + self.assertEqual(text, "") + self.assertEqual(error, "netveil-audit: usage_error\n") + self.assertNotIn(marker, error) + + def test_long_options_cannot_be_abbreviated(self) -> None: + marker = "PRIVATE-ABBREVIATION-MARKER" + for arguments in ( + ["--ver"], + ["receipt", f"{marker}-corpus", "--key-f", f"{marker}-key"], + ): + with self.subTest(arguments=arguments): + result, binary, text, error = self._run_main(arguments) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(binary, b"") + self.assertEqual(text, "") + self.assertEqual(error, "netveil-audit: usage_error\n") + self.assertNotIn(marker, error) + + def test_invalid_corpus_is_rejected_before_key_access_without_echo(self) -> None: + marker = "PRIVATE-ENDPOINT-MARKER" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / f"{marker}-corpus.txt" + missing_key = root / f"{marker}-missing-key.bin" + _write(corpus_path, f"{marker}:443\n".encode()) + + result, binary_output, text_output, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(missing_key), + ] + ) + + self.assertEqual(result, CliExitCode.CORPUS_REJECTED) + self.assertEqual(binary_output, b"") + self.assertEqual(text_output, "") + self.assertEqual( + error_output, + "netveil-audit: corpus_rejected:invalid_address:line=1\n", + ) + self.assertNotIn(marker, error_output) + + def test_empty_corpus_error_has_no_synthetic_line_number(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "empty.txt" + _write(corpus_path, b"") + + result, _, _, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(root / "unused-key"), + ] + ) + + self.assertEqual(result, CliExitCode.CORPUS_REJECTED) + self.assertEqual( + error_output, + "netveil-audit: corpus_rejected:empty_corpus\n", + ) + + def test_key_access_and_permission_failures_are_stable_and_redacted(self) -> None: + marker = "PRIVATE-KEY-PATH" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "corpus.txt" + _write(corpus_path, _DEMO_CORPUS) + + missing_key = root / f"{marker}-missing" + result, _, _, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(missing_key), + ] + ) + self.assertEqual(result, CliExitCode.KEY_UNAVAILABLE) + self.assertEqual(error_output, "netveil-audit: key_unavailable\n") + self.assertNotIn(marker, error_output) + + insecure_key = root / f"{marker}-insecure" + _write(insecure_key, _DEMO_KEY, 0o644) + result, _, _, error_output = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(insecure_key), + ] + ) + self.assertEqual(result, CliExitCode.KEY_REJECTED) + self.assertEqual(error_output, "netveil-audit: key_rejected\n") + self.assertNotIn(marker, error_output) + + def test_corpus_cannot_be_reused_or_copied_as_its_own_key(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "corpus.txt" + copied_key_path = root / "copied-key.bin" + _write(corpus_path, _DEMO_CORPUS) + _write(copied_key_path, _DEMO_CORPUS) + + for key_path in (corpus_path, copied_key_path): + with self.subTest(key_path=key_path.name): + result, binary, text, error = self._run_main( + [ + "receipt", + str(corpus_path), + "--key-file", + str(key_path), + ] + ) + self.assertEqual(result, CliExitCode.KEY_REJECTED) + self.assertEqual(binary, b"") + self.assertEqual(text, "") + self.assertEqual(error, "netveil-audit: key_rejected\n") + + def test_usage_failure_never_echoes_unrecognized_argument(self) -> None: + marker = "PRIVATE-COMMAND-LINE-MARKER" + result, binary_output, text_output, error_output = self._run_main( + [f"--unknown={marker}"] + ) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(binary_output, b"") + self.assertEqual(text_output, "") + self.assertEqual(error_output, "netveil-audit: usage_error\n") + self.assertNotIn(marker, error_output) + + def test_missing_command_is_a_stable_usage_failure(self) -> None: + result, _, _, error_output = self._run_main([]) + self.assertEqual(result, CliExitCode.USAGE) + self.assertEqual(error_output, "netveil-audit: usage_error\n") + + def test_help_returns_success_without_leaving_main(self) -> None: + result, binary_output, text_output, error_output = self._run_main(["--help"]) + self.assertEqual(result, CliExitCode.SUCCESS) + self.assertEqual(binary_output, b"") + self.assertIn("usage: netveil-audit", text_output) + self.assertEqual(error_output, "") + + def test_artifact_failure_happens_before_argument_or_path_handling(self) -> None: + marker = "PRIVATE-PATH-MARKER" + stdout = _CapturedStdout() + stderr = io.StringIO() + with ( + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified([f"--unknown={marker}"]) + self.assertEqual(result, CliExitCode.ARTIFACT_UNVERIFIED) + self.assertEqual(stderr.getvalue(), "netveil-audit: artifact_unverified\n") + self.assertNotIn(marker, stderr.getvalue()) + + def test_unexpected_failure_and_interrupt_are_redacted(self) -> None: + for failure, expected in ( + (RuntimeError("PRIVATE-INTERNAL-MARKER"), "internal_error"), + (KeyboardInterrupt(), "interrupted"), + ): + with self.subTest(expected=expected): + stdout = _CapturedStdout() + stderr = io.StringIO() + with ( + patch.object( + cli, + "_parser", + side_effect=failure, + ), + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified( + [], + verified_distribution_version=cli._DISTRIBUTION_VERSION, + ) + self.assertEqual(result, CliExitCode.INTERNAL_ERROR) + self.assertEqual(stderr.getvalue(), f"netveil-audit: {expected}\n") + self.assertNotIn("PRIVATE-INTERNAL-MARKER", stderr.getvalue()) + + def test_output_failure_is_redacted(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + corpus_path = root / "corpus.txt" + key_path = root / "key.bin" + _write(corpus_path, _DEMO_CORPUS) + _write(key_path, _DEMO_KEY) + stderr = io.StringIO() + stdout = _CapturedStdout() + with ( + patch.object( + stdout.buffer, + "write", + side_effect=BrokenPipeError("PRIVATE-OUTPUT-MARKER"), + ), + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified( + [ + "receipt", + str(corpus_path), + "--key-file", + str(key_path), + ], + verified_distribution_version=cli._DISTRIBUTION_VERSION, + ) + self.assertEqual(result, CliExitCode.OUTPUT_FAILED) + self.assertEqual(stderr.getvalue(), "netveil-audit: output_failed\n") + self.assertNotIn("PRIVATE-OUTPUT-MARKER", stderr.getvalue()) + + def test_short_version_write_is_an_output_failure(self) -> None: + stdout = _CapturedStdout() + stderr = io.StringIO() + with ( + patch.object(stdout, "write", return_value=0), + patch.object(sys, "stdout", cast(TextIO, stdout)), + patch.object(sys, "stderr", stderr), + ): + result = cli._main_verified( + ["--version"], + verified_distribution_version=cli._DISTRIBUTION_VERSION, + ) + self.assertEqual(result, CliExitCode.OUTPUT_FAILED) + self.assertEqual(stderr.getvalue(), "netveil-audit: output_failed\n") + + +@unittest.skipUnless(os.name == "posix", "safe file policy requires POSIX") +class CliFileBoundaryTests(unittest.TestCase): + def test_accepts_exact_regular_owner_only_key(self) -> None: + with tempfile.TemporaryDirectory() as directory: + key_path = Path(directory) / "key.bin" + _write(key_path, _DEMO_KEY, 0o600) + observed = cli._read_bounded_file( + key_path, + maximum_bytes=cli._MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + self.assertEqual(observed.payload, _DEMO_KEY) + + def test_rejects_insecure_short_linked_and_wrong_owner_keys(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + cases: list[tuple[str, bytes, int]] = [ + ("short", b"x" * 31, 0o600), + ("group-readable", _DEMO_KEY, 0o640), + ("owner-executable", _DEMO_KEY, 0o700), + ] + for name, payload, mode in cases: + with self.subTest(name=name): + key_path = root / name + _write(key_path, payload, mode) + with self.assertRaises(cli._CliFailure) as raised: + cli._read_bounded_file( + key_path, + maximum_bytes=cli._MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + self.assertEqual(raised.exception.code, "key_rejected") + key_path.unlink() + + linked = root / "linked" + alias = root / "alias" + _write(linked, _DEMO_KEY) + os.link(linked, alias) + with self.assertRaises(cli._CliFailure) as linked_failure: + cli._read_bounded_file( + linked, + maximum_bytes=cli._MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + self.assertEqual(linked_failure.exception.code, "key_rejected") + + linked.unlink() + alias.unlink() + wrong_owner = root / "wrong-owner" + _write(wrong_owner, _DEMO_KEY) + current_uid = wrong_owner.stat().st_uid + with ( + patch.object(os, "geteuid", return_value=current_uid + 1), + self.assertRaises(cli._CliFailure) as owner_failure, + ): + cli._read_bounded_file( + wrong_owner, + maximum_bytes=cli._MAX_KEY_BYTES, + failure_code="key_unavailable", + failure_exit=CliExitCode.KEY_UNAVAILABLE, + key_policy=True, + ) + self.assertEqual(owner_failure.exception.code, "key_rejected") + + def test_rejects_symlink_directory_fifo_missing_and_oversized_files(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / "target" + _write(target, _DEMO_KEY) + symlink = root / "symlink" + symlink.symlink_to(target) + fifo = root / "fifo" + os.mkfifo(fifo) + oversized = root / "oversized" + _write(oversized, b"x" * 33) + + for name, path, maximum in ( + ("symlink", symlink, cli._MAX_KEY_BYTES), + ("directory", root, cli._MAX_KEY_BYTES), + ("fifo", fifo, cli._MAX_KEY_BYTES), + ("missing", root / "missing", cli._MAX_KEY_BYTES), + ("oversized", oversized, 32), + ): + with ( + self.subTest(name=name), + self.assertRaises(cli._CliFailure) as raised, + ): + cli._read_bounded_file( + path, + maximum_bytes=maximum, + failure_code="file_unavailable", + failure_exit=CliExitCode.CORPUS_UNAVAILABLE, + key_policy=False, + ) + self.assertEqual(raised.exception.code, "file_unavailable") + self.assertIsNone(raised.exception.__context__) + + def test_detects_file_identity_change_and_close_failure(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input" + _write(path, b"abc") + status = path.stat() + changed = os.stat_result( + ( + status.st_mode, + status.st_ino, + status.st_dev, + status.st_nlink, + status.st_uid, + status.st_gid, + status.st_size + 1, + status.st_atime, + status.st_mtime, + status.st_ctime, + ) + ) + with ( + patch.object( + cli, + "_read_open_file", + return_value=(b"abc", status, changed), + ), + self.assertRaises(cli._CliFailure), + ): + cli._read_bounded_file( + path, + maximum_bytes=10, + failure_code="changed", + failure_exit=CliExitCode.CORPUS_UNAVAILABLE, + key_policy=False, + ) + + real_close = os.close + + def close_then_fail(descriptor: int) -> None: + real_close(descriptor) + raise OSError("PRIVATE-CLOSE-MARKER") + + with ( + patch.object(os, "close", side_effect=close_then_fail), + self.assertRaises(cli._CliFailure) as close_failure, + ): + cli._read_bounded_file( + path, + maximum_bytes=10, + failure_code="close_failed", + failure_exit=CliExitCode.CORPUS_UNAVAILABLE, + key_policy=False, + ) + self.assertEqual(close_failure.exception.code, "close_failed") + self.assertIsNone(close_failure.exception.__context__) + + def test_low_level_read_failure_returns_only_empty_sentinels(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input" + _write(path, b"abc") + descriptor = os.open(path, os.O_RDONLY) + try: + with patch.object(os, "fstat", side_effect=OSError("PRIVATE")): + result = cli._read_open_file(descriptor, maximum_bytes=10) + finally: + os.close(descriptor) + self.assertEqual(result, (None, None, None)) + + def test_low_level_read_stops_at_bound_without_waiting_for_eof(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "input" + _write(path, b"abc") + descriptor = os.open(path, os.O_RDONLY) + try: + current = os.fstat(descriptor) + before_growth = os.stat_result( + ( + current.st_mode, + current.st_ino, + current.st_dev, + current.st_nlink, + current.st_uid, + current.st_gid, + 2, + current.st_atime, + current.st_mtime, + current.st_ctime, + ) + ) + with patch.object( + os, + "fstat", + side_effect=(before_growth, current), + ): + payload, before, after = cli._read_open_file( + descriptor, + maximum_bytes=2, + ) + finally: + os.close(descriptor) + self.assertEqual(payload, b"abc") + self.assertIsNotNone(before) + self.assertIsNotNone(after) + + def test_unsupported_platform_fails_before_open(self) -> None: + with ( + patch.object(os, "name", "unsupported"), + self.assertRaises(cli._CliFailure) as raised, + ): + cli._open_flags() + self.assertEqual(raised.exception.code, "platform_unsupported") + + +class CliOutputBoundaryTests(unittest.TestCase): + def test_emit_failure_survives_broken_stderr(self) -> None: + stderr = io.StringIO() + with ( + patch.object( + stderr, + "write", + side_effect=OSError("PRIVATE-STDERR-MARKER"), + ), + patch.object(sys, "stderr", stderr), + ): + result = cli._emit_failure( + cli._CliFailure("safe", CliExitCode.CORPUS_REJECTED) + ) + self.assertEqual(result, CliExitCode.OUTPUT_FAILED) + + def test_parser_exit_messages_use_bounded_writes(self) -> None: + parser = cli._SafeArgumentParser(prog="netveil-audit") + stderr = io.StringIO() + with ( + patch.object(stderr, "write", return_value=0), + self.assertRaises(cli._CliFailure) as output_failure, + ): + parser._print_message("help", stderr) + self.assertEqual(output_failure.exception.code, "output_failed") + + with ( + patch.object(sys, "stderr", stderr), + self.assertRaises(cli._CliFailure) as usage_failure, + ): + parser.exit(2, "usage failed\n") + self.assertEqual(usage_failure.exception.code, "usage_error") + + stdout = io.StringIO() + with ( + patch.object(sys, "stdout", stdout), + self.assertRaises(cli._CliCompletion) as completion, + ): + parser.exit(0, "done\n") + self.assertEqual(completion.exception.exit_code, CliExitCode.SUCCESS) + self.assertEqual(stdout.getvalue(), "done\n") + + def test_exact_writers_reject_zero_and_accept_partial_progress(self) -> None: + text = io.StringIO() + binary = io.BytesIO() + with patch.object( + text, + "write", + side_effect=lambda value: min(2, len(value)), + ) as text_write: + self.assertTrue(cli._write_text(text, "abcdef")) + with patch.object( + binary, + "write", + side_effect=lambda value: min(2, len(value)), + ) as binary_write: + self.assertTrue(cli._write_binary(binary, b"abcdef")) + self.assertEqual(text_write.call_count, 3) + self.assertEqual(binary_write.call_count, 3) + + with ( + patch.object(text, "write", return_value=0), + patch.object(binary, "write", return_value=0), + ): + self.assertFalse(cli._write_text(text, "x")) + self.assertFalse(cli._write_binary(binary, b"x")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_evidence_renderer.py b/tests/test_evidence_renderer.py new file mode 100644 index 0000000..9448d0a --- /dev/null +++ b/tests/test_evidence_renderer.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +import hashlib +import json +import unittest +from xml.etree import ElementTree + +from tools import render_evidence +from tools import verify_fresh_wheel as verifier + +SOURCE_COMMIT = "a" * 40 +SOURCE_DATE_EPOCH = 1_700_000_000 + + +def _receipt() -> dict[str, object]: + report: dict[str, object] = { + "counts": { + "endpoint_occurrences": 5, + "physical_lines": 6, + "source_bytes": len(verifier._CORPUS), + "unique_endpoints": 4, + }, + "duplicates": {"group_count": 1}, + "endpoint_occurrences_by_ip_version": {"ipv4": 4, "ipv6": 1}, + "endpoint_occurrences_by_scope": {"documentation": 5}, + "source_content_id": "nvs1_" + "b" * 64, + } + return { + "report": report, + "report_digest": { + "algorithm": "sha256", + "value": hashlib.sha256( + render_evidence._canonical_json(report) + ).hexdigest(), + }, + "schema": "netveil.aggregate-receipt.v1", + } + + +def _documents() -> tuple[dict[str, object], dict[str, object]]: + artifacts: list[object] = [ + { + "filename": "netveil_audit-0.3.0-py3-none-any.whl", + "kind": "wheel", + "sha256": "c" * 64, + "size_bytes": 24_000, + }, + { + "filename": "netveil_audit-0.3.0.tar.gz", + "kind": "sdist", + "members": [ + { + "kind": "directory", + "mode": "0755", + "path": "netveil_audit-0.3.0", + "size_bytes": 0, + }, + { + "kind": "file", + "mode": "0644", + "path": "netveil_audit-0.3.0/README.md", + "sha256": "d" * 64, + "size_bytes": 100, + }, + ], + "sha256": "e" * 64, + "size_bytes": 31_000, + }, + ] + inventory = { + "artifacts": artifacts, + "schema": render_evidence.INVENTORY_SCHEMA, + "source_commit": SOURCE_COMMIT, + "source_date_epoch": SOURCE_DATE_EPOCH, + } + inventory_payload = render_evidence._json_bytes(inventory) + receipt = _receipt() + receipt_payload = render_evidence._json_bytes(receipt) + verification: dict[str, object] = { + "checks": [{"name": name, "status": "pass"} for name in verifier._CHECKS], + "installed": { + "launcher": { + "logical_path": "bin/netveil-audit", + "mode": "0755", + "sha256": "f" * 64, + "size_bytes": 8_000, + }, + "record": { + "logical_path": ("site-packages/netveil_audit-0.3.0.dist-info/RECORD"), + "mode": "0644", + "sha256": "1" * 64, + "size_bytes": 2_000, + }, + "selected_record_rows": [ + { + "path": "../../../bin/netveil-audit", + "sha256": "f" * 64, + "size_bytes": 8_000, + }, + { + "path": "netveil_audit-0.3.0.dist-info/RECORD", + "sha256": None, + "size_bytes": None, + }, + ], + }, + "integrity_evidence": { + "artifacts": artifacts, + "attestation_verified": False, + "inventory_schema": render_evidence.INVENTORY_SCHEMA, + "inventory_sha256": hashlib.sha256(inventory_payload).hexdigest(), + "inventory_type": "unsigned_sha256_manifest", + "signature_verified": False, + "source_commit": SOURCE_COMMIT, + "source_date_epoch": SOURCE_DATE_EPOCH, + }, + "interpreter": { + "cache_tag": "cpython-312", + "implementation": "cpython", + "version": "3.12.3", + }, + "platform": { + "machine": "x86_64", + "release": "test-kernel", + "sys_platform": "linux", + "system": "Linux", + }, + "public_demo": { + "classification": ( + "synthetic_ietf_documentation_ranges_with_public_demo_key" + ), + "commands": [ + { + "argv": ["netveil-audit", "--version"], + "exit_code": 0, + "stderr": "", + "stdout": "netveil-audit 0.3.0\n", + }, + { + "argv": [ + "netveil-audit", + "receipt", + "documentation-corpus.txt", + "--key-file", + "public-demo.key", + ], + "exit_code": 0, + "stderr": "", + "stdout_json": receipt, + "stdout_sha256": hashlib.sha256(receipt_payload).hexdigest(), + }, + ], + "corpus": { + "physical_lines": 6, + "sha256": hashlib.sha256(verifier._CORPUS).hexdigest(), + "size_bytes": len(verifier._CORPUS), + }, + "public_demo_key": { + "classification": "public_non_secret_test_material", + "sha256": hashlib.sha256(verifier._PUBLIC_DEMO_KEY).hexdigest(), + "size_bytes": 32, + "source_constant": ("tools/verify_fresh_wheel.py:_PUBLIC_DEMO_KEY"), + }, + }, + "schema": render_evidence.VERIFICATION_SCHEMA, + "source_commit": SOURCE_COMMIT, + "status": "pass", + "syscall_traces": [ + { + "exec_chain": ["installed_launcher", "installed_python"], + "exec_count": 2, + "exit_syscall_count": 1, + "label": label, + "network_syscall_count": 0, + "normalized_sha256": digest * 64, + "post_launch_process_count": 0, + "process_count": 1, + } + for label, digest in (("receipt", "2"), ("version", "3")) + ], + "wheel": { + "members": [ + { + "mode": "0644", + "path": "netveil/cli.py", + "sha256": "4" * 64, + "size_bytes": 7_000, + } + ], + "sha256": "c" * 64, + "size_bytes": 24_000, + }, + } + return verification, inventory + + +def _payloads() -> tuple[bytes, bytes]: + verification, inventory = _documents() + return ( + render_evidence._json_bytes(verification), + render_evidence._json_bytes(inventory), + ) + + +class EvidenceRendererTests(unittest.TestCase): + def test_renders_every_visual_from_cross_bound_evidence(self) -> None: + verification, inventory = _payloads() + + outputs = render_evidence.render_bundle( + verification, + inventory, + generator_payload=b"renderer source fixture\n", + ) + + self.assertEqual(set(outputs), set(render_evidence.ALL_OUTPUT_PATHS)) + for path in ( + render_evidence.CLI_SVG_PATH, + render_evidence.COUNTS_SVG_PATH, + render_evidence.MATRIX_SVG_PATH, + render_evidence.PROVENANCE_SVG_PATH, + ): + ElementTree.fromstring(outputs[path]) + self.assertIn(b"captured stdout", outputs[render_evidence.CLI_SVG_PATH]) + self.assertIn(b"endpoint occurrences", outputs[render_evidence.COUNTS_SVG_PATH]) + self.assertIn( + b"coordinated bootstrap record mutation accepted", + outputs[render_evidence.MATRIX_SVG_PATH], + ) + self.assertIn( + SOURCE_COMMIT.encode(), + outputs[render_evidence.PROVENANCE_SVG_PATH], + ) + self.assertIn( + b"netveil-audit receipt documentation-corpus.txt", + outputs[render_evidence.CAST_PATH], + ) + + manifest = json.loads(outputs[render_evidence.MANIFEST_PATH]) + self.assertEqual( + manifest["schema"], + render_evidence.VISUAL_MANIFEST_SCHEMA, + ) + self.assertEqual(manifest["source_commit"], SOURCE_COMMIT) + self.assertEqual( + {record["path"] for record in manifest["outputs"]}, + set(render_evidence.VISUAL_OUTPUT_PATHS), + ) + for record in manifest["outputs"]: + self.assertEqual( + record["sha256"], + hashlib.sha256(outputs[record["path"]]).hexdigest(), + ) + + def test_rendering_is_byte_deterministic(self) -> None: + verification, inventory = _payloads() + + first = render_evidence.render_bundle( + verification, + inventory, + generator_payload=b"same generator\n", + ) + second = render_evidence.render_bundle( + verification, + inventory, + generator_payload=b"same generator\n", + ) + + self.assertEqual(first, second) + + def test_rejects_noncanonical_or_cross_document_drift(self) -> None: + verification, inventory = _documents() + noncanonical = ( + json.dumps(verification, indent=2, sort_keys=True).encode("ascii") + b"\n" + ) + with self.assertRaisesRegex( + render_evidence.EvidenceRenderError, + "not canonical", + ): + render_evidence.parse_evidence( + noncanonical, + render_evidence._json_bytes(inventory), + ) + + verification["source_commit"] = "b" * 40 + with self.assertRaisesRegex( + render_evidence.EvidenceRenderError, + "source commits differ", + ): + render_evidence.parse_evidence( + render_evidence._json_bytes(verification), + render_evidence._json_bytes(inventory), + ) + + def test_rejects_raw_endpoint_text_and_receipt_digest_drift(self) -> None: + verification, inventory = _documents() + verification["unexpected_raw_endpoint"] = "192.0.2.10" + with self.assertRaisesRegex( + render_evidence.EvidenceRenderError, + "raw endpoint", + ): + render_evidence.parse_evidence( + render_evidence._json_bytes(verification), + render_evidence._json_bytes(inventory), + ) + + verification, inventory = _documents() + demo = verification["public_demo"] + assert isinstance(demo, dict) + commands = demo["commands"] + assert isinstance(commands, list) + receipt_command = commands[1] + assert isinstance(receipt_command, dict) + receipt_command["stdout_sha256"] = "0" * 64 + with self.assertRaisesRegex( + render_evidence.EvidenceRenderError, + "receipt stdout digest", + ): + render_evidence.parse_evidence( + render_evidence._json_bytes(verification), + render_evidence._json_bytes(inventory), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_fresh_wheel_verifier.py b/tests/test_fresh_wheel_verifier.py new file mode 100644 index 0000000..e3db31a --- /dev/null +++ b/tests/test_fresh_wheel_verifier.py @@ -0,0 +1,924 @@ +from __future__ import annotations + +import csv +import gzip +import hashlib +import importlib.util +import io +import json +import marshal +import stat +import tarfile +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path +from types import CodeType +from unittest.mock import patch +from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo + +from tools import verify_fresh_wheel as verifier + +_SOURCE_COMMIT = "a" * 40 +_SOURCE_DATE_EPOCH = 1_700_000_000 +_LAUNCHER = ( + b"#!/bin/sh\n" + b'""":"\n' + b'case "$0" in\n' + b" */*) netveil_script_directory=${0%/*} ;;\n" + b" *) exit 70 ;;\n" + b"esac\n" + b'exec "$netveil_script_directory/python" -IESB "$0" "$@"\n' + b"exit 70\n" + b'":"""\n' + b"\n" + b"raise SystemExit(0)\n" +) + + +def _record_payload(members: dict[str, bytes]) -> bytes: + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + record_name = f"{verifier._DIST_INFO}/RECORD" + for name in sorted((*members, record_name)): + if name == record_name: + writer.writerow((name, "", "")) + else: + payload = members[name] + writer.writerow( + ( + name, + f"sha256={verifier._record_digest(payload)}", + str(len(payload)), + ) + ) + return output.getvalue().encode() + + +def _wheel_members() -> dict[str, bytes]: + members = { + verifier._WHEEL_SCRIPT: _LAUNCHER, + "netveil/__init__.py": b"", + "netveil/cli.py": b"", + "netveil/model.py": b"", + "netveil/parser.py": b"", + "netveil/privacy.py": b"", + "netveil/py.typed": b"", + verifier._BOOTSTRAP_NAME: b"", + f"{verifier._DIST_INFO}/METADATA": ( + b"Metadata-Version: 2.4\n" + b"Name: netveil-audit\n" + b"Version: 0.3.0\n" + b"Requires-Python: >=3.11\n" + ), + f"{verifier._DIST_INFO}/WHEEL": ( + b"Wheel-Version: 1.0\n" + b"Generator: verifier-test\n" + b"Root-Is-Purelib: true\n" + b"Tag: py3-none-any\n" + ), + f"{verifier._DIST_INFO}/licenses/LICENSE": b"test-only license\n", + f"{verifier._DIST_INFO}/top_level.txt": (b"netveil\nnetveil_bootstrap\n"), + } + members[f"{verifier._DIST_INFO}/RECORD"] = _record_payload(members) + return members + + +def _zip_bytes(members: dict[str, bytes]) -> bytes: + output = io.BytesIO() + with ZipFile(output, "w") as archive: + for name, payload in members.items(): + info = ZipInfo(name) + info.create_system = 3 + permissions = 0o755 if name == verifier._WHEEL_SCRIPT else 0o644 + info.external_attr = (stat.S_IFREG | permissions) << 16 + info.compress_type = ZIP_DEFLATED + archive.writestr(info, payload) + return output.getvalue() + + +def _installed_record( + *, + site_root: Path, + launcher: Path, + launcher_payload: bytes, +) -> bytes: + relative = verifier._installed_record_path(site_root, launcher) + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + writer.writerow( + ( + relative, + f"sha256={verifier._record_digest(launcher_payload)}", + str(len(launcher_payload)), + ) + ) + writer.writerow((f"{verifier._DIST_INFO}/RECORD", "", "")) + return output.getvalue().encode() + + +def _receipt(key: bytes) -> bytes: + report: dict[str, object] = { + "counts": { + "endpoint_occurrences": 5, + "physical_lines": 6, + "source_bytes": len(verifier._CORPUS), + "unique_endpoints": 4, + }, + "duplicates": {"group_count": 1}, + "endpoint_occurrences_by_ip_version": {"ipv4": 4, "ipv6": 1}, + "endpoint_occurrences_by_scope": {"documentation": 5}, + "source_content_id": "nvs1_" + "0" * 64, + } + document = { + "report": report, + "report_digest": { + "algorithm": "sha256", + "value": hashlib.sha256(verifier._canonical_json(report)).hexdigest(), + }, + "schema": "netveil.aggregate-receipt.v1", + } + payload = verifier._canonical_json(document) + b"\n" + if key in payload: + raise AssertionError("test fixture unexpectedly contains private key") + return payload + + +def _sdist_payload() -> bytes: + tar_payload = io.BytesIO() + with tarfile.open( + fileobj=tar_payload, + mode="w:", + format=tarfile.USTAR_FORMAT, + ) as archive: + directory = tarfile.TarInfo("netveil_audit-0.3.0/") + directory.type = tarfile.DIRTYPE + directory.mode = 0o755 + directory.mtime = _SOURCE_DATE_EPOCH + directory.uid = 0 + directory.gid = 0 + directory.uname = "" + directory.gname = "" + archive.addfile(directory) + + readme_payload = b"# Netveil test sdist\n" + readme = tarfile.TarInfo("netveil_audit-0.3.0/README.md") + readme.type = tarfile.REGTYPE + readme.mode = 0o644 + readme.mtime = _SOURCE_DATE_EPOCH + readme.uid = 0 + readme.gid = 0 + readme.uname = "" + readme.gname = "" + readme.size = len(readme_payload) + archive.addfile(readme, io.BytesIO(readme_payload)) + compressed = io.BytesIO() + with gzip.GzipFile( + filename="", + mode="wb", + compresslevel=9, + fileobj=compressed, + mtime=_SOURCE_DATE_EPOCH, + ) as archive: + archive.write(tar_payload.getvalue()) + return compressed.getvalue() + + +def _sdist_inventory_members() -> list[dict[str, object]]: + readme_payload = b"# Netveil test sdist\n" + return [ + { + "kind": "directory", + "mode": "0755", + "path": "netveil_audit-0.3.0", + "size_bytes": 0, + }, + { + "kind": "file", + "mode": "0644", + "path": "netveil_audit-0.3.0/README.md", + "sha256": hashlib.sha256(readme_payload).hexdigest(), + "size_bytes": len(readme_payload), + }, + ] + + +def _inventory_payload( + wheel_payload: bytes, + *, + source_commit: str = _SOURCE_COMMIT, + wheel_filename: str = verifier._WHEEL_NAME, + wheel_sha256: str | None = None, + wheel_size_bytes: int | None = None, + sdist_sha256: str | None = None, +) -> bytes: + sdist_payload = _sdist_payload() + artifacts = [ + { + "filename": wheel_filename, + "kind": "wheel", + "sha256": ( + hashlib.sha256(wheel_payload).hexdigest() + if wheel_sha256 is None + else wheel_sha256 + ), + "size_bytes": ( + len(wheel_payload) if wheel_size_bytes is None else wheel_size_bytes + ), + }, + { + "filename": verifier._SDIST_NAME, + "kind": "sdist", + "members": _sdist_inventory_members(), + "sha256": ( + hashlib.sha256(sdist_payload).hexdigest() + if sdist_sha256 is None + else sdist_sha256 + ), + "size_bytes": len(sdist_payload), + }, + ] + artifacts.sort(key=lambda artifact: str(artifact["filename"])) + return ( + verifier._canonical_json( + { + "artifacts": artifacts, + "schema": verifier._INVENTORY_SCHEMA, + "source_commit": source_commit, + "source_date_epoch": _SOURCE_DATE_EPOCH, + } + ) + + b"\n" + ) + + +class WheelArchiveContractTests(unittest.TestCase): + def test_accepts_exact_inventory_and_record_bound_launcher(self) -> None: + payload = _zip_bytes(_wheel_members()) + + evidence = verifier._inspect_wheel(payload) + + self.assertEqual(evidence.payload, payload) + self.assertEqual(evidence.launcher, _LAUNCHER) + self.assertEqual(evidence.sha256, hashlib.sha256(payload).hexdigest()) + self.assertEqual( + [member.path for member in evidence.members], + sorted(_wheel_members()), + ) + self.assertTrue( + all(member.mode in ("0644", "0755") for member in evidence.members) + ) + + def test_rejects_entry_points_even_when_recorded(self) -> None: + members = _wheel_members() + record_name = f"{verifier._DIST_INFO}/RECORD" + members.pop(record_name) + members[f"{verifier._DIST_INFO}/entry_points.txt"] = ( + b"[console_scripts]\nnetveil-audit=netveil_bootstrap:entrypoint\n" + ) + members[record_name] = _record_payload(members) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^wheel_inventory_invalid$", + ): + verifier._inspect_wheel(_zip_bytes(members)) + + def test_rejects_stale_record_hash(self) -> None: + members = _wheel_members() + members["netveil/cli.py"] = b"# unrecorded mutation\n" + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^record_mismatch$", + ): + verifier._inspect_wheel(_zip_bytes(members)) + + def test_rejects_duplicate_record_paths(self) -> None: + digest = verifier._record_digest(b"x") + payload = ( + f"netveil/cli.py,sha256={digest},1\nnetveil/cli.py,sha256={digest},1\n" + ).encode() + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^record_invalid$", + ): + verifier._parse_record(payload) + + def test_exact_wheel_reader_rejects_symlink(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + target = root / "target.whl" + target.write_bytes(b"wheel") + link = root / verifier._WHEEL_NAME + link.symlink_to(target) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^wheel_path_invalid$", + ): + verifier._read_exact_wheel(link) + + +class ReleaseInventoryContractTests(unittest.TestCase): + def test_accepts_exact_canonical_unsigned_inventory(self) -> None: + wheel_payload = b"test wheel bytes" + payload = _inventory_payload(wheel_payload) + + inventory = verifier._parse_release_inventory(payload) + + self.assertEqual(inventory.source_commit, _SOURCE_COMMIT) + self.assertEqual(inventory.source_date_epoch, 1_700_000_000) + self.assertEqual(inventory.sha256, hashlib.sha256(payload).hexdigest()) + self.assertEqual( + {artifact.kind for artifact in inventory.artifacts}, + {"wheel", "sdist"}, + ) + + def test_rejects_malformed_and_noncanonical_inventory(self) -> None: + canonical = _inventory_payload(b"test wheel bytes") + document = json.loads(canonical) + malformed_document = dict(document) + malformed_document.pop("source_commit") + malformed = verifier._canonical_json(malformed_document) + b"\n" + noncanonical = json.dumps(document, indent=2).encode("ascii") + b"\n" + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_invalid$", + ): + verifier._parse_release_inventory(malformed) + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_not_canonical$", + ): + verifier._parse_release_inventory(noncanonical) + + def test_rejects_duplicate_inventory_keys(self) -> None: + payload = _inventory_payload(b"test wheel bytes") + duplicated = payload.replace( + b'{"artifacts":', + b'{"schema":"netveil.release-inventory.v1","artifacts":', + 1, + ) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_invalid$", + ): + verifier._parse_release_inventory(duplicated) + + def test_rejects_commit_name_size_and_sha256_mismatches(self) -> None: + wheel_payload = b"not required to be a valid wheel for binding failures" + cases = ( + ( + "commit", + _inventory_payload(wheel_payload), + "b" * 40, + "inventory_source_commit_mismatch", + ), + ( + "name", + _inventory_payload( + wheel_payload, + wheel_filename="unexpected-0.3.0-py3-none-any.whl", + ), + _SOURCE_COMMIT, + "inventory_wheel_name_mismatch", + ), + ( + "size", + _inventory_payload( + wheel_payload, + wheel_size_bytes=len(wheel_payload) + 1, + ), + _SOURCE_COMMIT, + "inventory_wheel_size_mismatch", + ), + ( + "sha256", + _inventory_payload( + wheel_payload, + wheel_sha256="b" * 64, + ), + _SOURCE_COMMIT, + "inventory_wheel_sha256_mismatch", + ), + ( + "sdist_sha256", + _inventory_payload( + wheel_payload, + sdist_sha256="b" * 64, + ), + _SOURCE_COMMIT, + "inventory_sdist_sha256_mismatch", + ), + ) + for label, inventory_payload, expected_commit, failure_code in cases: + with self.subTest(label=label), tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + wheel = root / verifier._WHEEL_NAME + sdist = root / verifier._SDIST_NAME + inventory = root / verifier._INVENTORY_NAME + wheel.write_bytes(wheel_payload) + sdist.write_bytes(_sdist_payload()) + inventory.write_bytes(inventory_payload) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + f"^{failure_code}$", + ): + verifier.verify_wheel( + wheel, + inventory_path=inventory, + sdist_path=sdist, + source_commit=expected_commit, + ) + + def test_sdist_member_inventory_matches_safe_canonical_archive(self) -> None: + payload = _sdist_payload() + inventory = verifier._parse_release_inventory( + _inventory_payload(b"test wheel bytes") + ) + actual = verifier._inspect_sdist( + payload, + source_date_epoch=_SOURCE_DATE_EPOCH, + ) + + verifier._bind_sdist_members(inventory, actual) + self.assertEqual( + [member.path for member in actual], + [ + "netveil_audit-0.3.0", + "netveil_audit-0.3.0/README.md", + ], + ) + + def test_sdist_member_inventory_mismatch_is_rejected(self) -> None: + payload = _inventory_payload(b"test wheel bytes") + document = json.loads(payload) + sdist = next( + artifact + for artifact in document["artifacts"] + if artifact["kind"] == "sdist" + ) + sdist["members"][1]["sha256"] = "b" * 64 + inventory = verifier._parse_release_inventory( + verifier._canonical_json(document) + b"\n" + ) + actual = verifier._inspect_sdist( + _sdist_payload(), + source_date_epoch=_SOURCE_DATE_EPOCH, + ) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_sdist_members_mismatch$", + ): + verifier._bind_sdist_members(inventory, actual) + + def test_windows_drive_like_sdist_members_are_rejected_consistently( + self, + ) -> None: + payload = _inventory_payload(b"test wheel bytes") + document = json.loads(payload) + sdist = next( + artifact + for artifact in document["artifacts"] + if artifact["kind"] == "sdist" + ) + sdist["members"][0]["path"] = "C:escape" + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^inventory_invalid$", + ): + verifier._parse_release_inventory( + verifier._canonical_json(document) + b"\n" + ) + + member = tarfile.TarInfo("C:escape") + member.type = tarfile.REGTYPE + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^sdist_member_unsafe$", + ): + verifier._safe_sdist_member_name(member) + + +class InstalledLayoutContractTests(unittest.TestCase): + def create_layout(self, root: Path) -> tuple[Path, Path]: + prefix = root / "venv" + binary = prefix / "bin" + site_root = prefix / "lib" / "python3.11" / "site-packages" + dist_info = site_root / verifier._DIST_INFO + package = site_root / "netveil" + binary.mkdir(parents=True) + dist_info.mkdir(parents=True) + package.mkdir() + python = binary / "python" + python.write_bytes(b"test interpreter placeholder") + launcher = binary / verifier._LAUNCHER_NAME + launcher.write_bytes(_LAUNCHER) + launcher.chmod(0o755) + (site_root / verifier._BOOTSTRAP_NAME).write_bytes(b"") + (package / "cli.py").write_bytes(b"") + (dist_info / "METADATA").write_bytes( + b"Metadata-Version: 2.4\nName: netveil-audit\nVersion: 0.3.0\n" + ) + (dist_info / "RECORD").write_bytes( + _installed_record( + site_root=site_root, + launcher=launcher, + launcher_payload=_LAUNCHER, + ) + ) + return prefix, python + + def test_binds_installed_launcher_to_record_and_metadata(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + prefix, python = self.create_layout(root) + + layout = verifier._inspect_install(prefix, python, _LAUNCHER) + + self.assertEqual(layout.launcher.read_bytes(), _LAUNCHER) + self.assertEqual(layout.dist_info.name, verifier._DIST_INFO) + self.assertIsNotNone(layout.evidence) + assert layout.evidence is not None + self.assertEqual(layout.evidence.launcher.mode, "0755") + self.assertEqual( + layout.evidence.launcher.sha256, + hashlib.sha256(_LAUNCHER).hexdigest(), + ) + self.assertGreater(layout.evidence.record.size_bytes, 0) + self.assertEqual( + {row.path for row in layout.evidence.selected_record_rows}, + { + verifier._installed_record_path( + layout.site_root, + layout.launcher, + ), + f"{verifier._DIST_INFO}/RECORD", + }, + ) + + def test_rejects_installed_entry_points(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + root = Path(raw_root) + prefix, python = self.create_layout(root) + dist_info = next(prefix.glob("lib/python*/site-packages/*.dist-info")) + (dist_info / "entry_points.txt").write_bytes( + b"[console_scripts]\nnetveil-audit=netveil_bootstrap:entrypoint\n" + ) + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^entry_points_present$", + ): + verifier._inspect_install(prefix, python, _LAUNCHER) + + +class AdversarialPrimitiveTests(unittest.TestCase): + def test_record_rewrite_changes_only_selected_binding(self) -> None: + original = ( + b"netveil_bootstrap.py,sha256=" + + verifier._record_digest(b"old").encode() + + b",3\nother.py,sha256=" + + verifier._record_digest(b"same").encode() + + b",4\n" + ) + + updated = verifier._updated_record( + original, + path="netveil_bootstrap.py", + replacement=b"new bytes", + ) + records = verifier._parse_record(updated) + + self.assertEqual( + records["netveil_bootstrap.py"].digest, + verifier._record_digest(b"new bytes"), + ) + self.assertEqual( + records["other.py"].digest, + verifier._record_digest(b"same"), + ) + + def test_temporary_mutation_restores_bytes_and_mode_after_failure(self) -> None: + with tempfile.TemporaryDirectory() as raw_root: + path = Path(raw_root) / "artifact.py" + path.write_bytes(b"original") + path.chmod(0o640) + + with ( + self.assertRaisesRegex(RuntimeError, "^stop$"), + verifier._temporary_bytes(path, b"tampered"), + ): + self.assertEqual(path.read_bytes(), b"tampered") + raise RuntimeError("stop") + + self.assertEqual(path.read_bytes(), b"original") + self.assertEqual(stat.S_IMODE(path.stat().st_mode), 0o640) + + def test_unchecked_hash_pyc_contains_adversarial_code_object(self) -> None: + payload = verifier._unchecked_hash_pyc( + b"sentinel = 7\n", + b"trusted source\n", + ) + + self.assertEqual(payload[:4], importlib.util.MAGIC_NUMBER) + self.assertEqual(int.from_bytes(payload[4:8], "little"), 1) + code = marshal.loads(payload[16:]) + self.assertIsInstance(code, CodeType) + namespace: dict[str, object] = {} + exec(code, namespace) # noqa: S102 - executes fixed unit-test bytes. + self.assertEqual(namespace["sentinel"], 7) + + def test_receipt_validator_accepts_canonical_redacted_fixture(self) -> None: + key = b"k" * 32 + + document = verifier._verify_receipt_document(_receipt(key), key=key) + + self.assertEqual(document["schema"], "netveil.aggregate-receipt.v1") + + def test_public_demo_key_is_explicit_non_secret_test_material(self) -> None: + self.assertEqual(len(verifier._PUBLIC_DEMO_KEY), 32) + self.assertEqual( + hashlib.sha256(verifier._PUBLIC_DEMO_KEY).hexdigest(), + "2d27befbc438954c4a55d8c0e36192c5a4a7e9e3f15c17d6e427c4a3499d945d", + ) + + def test_receipt_validator_rejects_private_key_material(self) -> None: + key = b"k" * 32 + payload = _receipt(key) + document = json.loads(payload) + document["private"] = key.hex() + + with self.assertRaisesRegex( + verifier.VerificationFailure, + "^receipt_private_data_detected$", + ): + verifier._verify_receipt_document( + verifier._canonical_json(document) + b"\n", + key=key, + ) + + def test_syscall_trace_accepts_only_launcher_python_exec_chain(self) -> None: + root = Path("/private-verifier-root") + layout = verifier.InstalledLayout( + prefix=root / "venv", + python=root / "venv/bin/python", + launcher=root / "venv/bin/netveil-audit", + site_root=root / "venv/lib/python3.11/site-packages", + dist_info=root / "dist-info", + bootstrap=root / "bootstrap.py", + package_root=root / "netveil", + record=root / "RECORD", + ) + payload = ( + f'execve("{layout.launcher}", ["netveil-audit"], 0x0) = 0\n' + f'execve("{layout.python}", ["python", "-IESB"], 0x0) = 0\n' + "exit_group(0) = ?\n" + ).encode() + + evidence = verifier._validate_syscall_trace( + (payload,), + layout=layout, + label="version", + ) + + self.assertEqual( + evidence.exec_chain, + ("installed_launcher", "installed_python"), + ) + self.assertEqual(evidence.network_syscall_count, 0) + self.assertEqual(evidence.post_launch_process_count, 0) + self.assertEqual(len(evidence.normalized_sha256), 64) + + def test_syscall_trace_rejects_network_and_post_launch_processes(self) -> None: + root = Path("/private-verifier-root") + layout = verifier.InstalledLayout( + prefix=root / "venv", + python=root / "venv/bin/python", + launcher=root / "venv/bin/netveil-audit", + site_root=root / "venv/lib/python3.11/site-packages", + dist_info=root / "dist-info", + bootstrap=root / "bootstrap.py", + package_root=root / "netveil", + record=root / "RECORD", + ) + prefix = ( + f'execve("{layout.launcher}", ["netveil-audit"], 0x0) = 0\n' + f'execve("{layout.python}", ["python", "-IESB"], 0x0) = 0\n' + ) + for syscall in ( + "socket(AF_INET, SOCK_STREAM, IPPROTO_IP) = 3\n", + "clone(child_stack=NULL, flags=SIGCHLD) = 42\n", + ): + with ( + self.subTest(syscall=syscall), + self.assertRaisesRegex( + verifier.VerificationFailure, + "^network_or_process_activity_detected$", + ), + ): + verifier._validate_syscall_trace( + ((prefix + syscall + "exit_group(0) = ?\n").encode(),), + layout=layout, + ) + + def test_source_commit_must_be_exact_lowercase_sha1(self) -> None: + verifier._validate_source_commit(_SOURCE_COMMIT) + for invalid in ("a" * 39, "A" * 40, "g" * 40, "/private/path"): + with ( + self.subTest(invalid=invalid), + self.assertRaisesRegex( + verifier.VerificationFailure, + "^source_commit_invalid$", + ), + ): + verifier._validate_source_commit(invalid) + + +class VerifierCliTests(unittest.TestCase): + def test_success_json_contains_no_input_path(self) -> None: + stdout = io.StringIO() + stderr = io.StringIO() + summary = verifier.VerificationSummary( + source_commit=_SOURCE_COMMIT, + release_inventory=verifier._parse_release_inventory( + _inventory_payload(b"summary wheel bytes") + ), + installed=verifier.InstalledEvidence( + launcher=verifier.InstalledFileEvidence( + logical_path="bin/netveil-audit", + mode="0755", + sha256="c" * 64, + size_bytes=456, + ), + record=verifier.InstalledFileEvidence( + logical_path=("site-packages/netveil_audit-0.3.0.dist-info/RECORD"), + mode="0644", + sha256="d" * 64, + size_bytes=789, + ), + selected_record_rows=( + verifier.InstalledRecordRowEvidence( + path="../../../bin/netveil-audit", + sha256="c" * 64, + size_bytes=456, + ), + ), + ), + interpreter=verifier.InterpreterEvidence( + implementation="cpython", + version="3.11.0", + cache_tag="cpython-311", + ), + platform=verifier.PlatformEvidence( + sys_platform="linux", + system="Linux", + release="test-kernel", + machine="x86_64", + ), + syscall_traces=( + verifier.TraceEvidence( + label="version", + normalized_sha256="e" * 64, + process_count=1, + exec_chain=("installed_launcher", "installed_python"), + exec_count=2, + exit_syscall_count=1, + network_syscall_count=0, + post_launch_process_count=0, + ), + ), + public_demo=verifier.PublicDemoEvidence( + corpus_sha256=hashlib.sha256(verifier._CORPUS).hexdigest(), + corpus_size_bytes=len(verifier._CORPUS), + corpus_physical_lines=verifier._CORPUS.count(b"\n"), + public_key_sha256=hashlib.sha256(verifier._PUBLIC_DEMO_KEY).hexdigest(), + public_key_size_bytes=len(verifier._PUBLIC_DEMO_KEY), + version_stdout="netveil-audit 0.3.0\n", + receipt=json.loads(_receipt(verifier._PUBLIC_DEMO_KEY)), + receipt_stdout_sha256=hashlib.sha256( + _receipt(verifier._PUBLIC_DEMO_KEY) + ).hexdigest(), + ), + wheel_sha256="a" * 64, + wheel_size_bytes=123, + wheel_members=( + verifier.WheelMemberEvidence( + path="netveil/cli.py", + sha256="b" * 64, + size=7, + mode="0644", + ), + ), + ) + + with ( + patch.object(verifier, "verify_wheel", return_value=summary), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + result = verifier.main( + [ + "--source-commit", + _SOURCE_COMMIT, + "--inventory", + "/home/private/workspace/release-inventory.json", + "--sdist", + "/home/private/workspace/netveil_audit-0.3.0.tar.gz", + ("/home/private/workspace/netveil_audit-0.3.0-py3-none-any.whl"), + ] + ) + + self.assertEqual(result, 0) + self.assertEqual(stderr.getvalue(), "") + document = json.loads(stdout.getvalue()) + self.assertEqual(document["status"], "pass") + self.assertEqual(document["source_commit"], _SOURCE_COMMIT) + self.assertEqual( + document["integrity_evidence"]["inventory_type"], + "unsigned_sha256_manifest", + ) + self.assertFalse(document["integrity_evidence"]["signature_verified"]) + self.assertFalse(document["integrity_evidence"]["attestation_verified"]) + self.assertEqual(document["installed"]["launcher"]["mode"], "0755") + self.assertEqual(document["installed"]["record"]["size_bytes"], 789) + self.assertEqual(document["platform"]["machine"], "x86_64") + self.assertEqual( + document["public_demo"]["classification"], + "synthetic_ietf_documentation_ranges_with_public_demo_key", + ) + self.assertEqual( + document["public_demo"]["commands"][0]["stdout"], + "netveil-audit 0.3.0\n", + ) + self.assertEqual( + document["public_demo"]["public_demo_key"]["classification"], + "public_non_secret_test_material", + ) + self.assertEqual( + document["syscall_traces"][0]["exec_chain"], + ["installed_launcher", "installed_python"], + ) + self.assertEqual(document["interpreter"]["implementation"], "cpython") + self.assertEqual(document["wheel"]["members"][0]["path"], "netveil/cli.py") + self.assertNotIn("/home/private", stdout.getvalue()) + self.assertEqual( + [item["name"] for item in document["checks"]], + list(verifier._CHECKS), + ) + + def test_failure_is_stable_and_redacts_input_path(self) -> None: + stdout = io.StringIO() + stderr = io.StringIO() + + with ( + patch.object( + verifier, + "verify_wheel", + side_effect=verifier.VerificationFailure("wheel_path_invalid"), + ), + redirect_stdout(stdout), + redirect_stderr(stderr), + ): + result = verifier.main( + [ + "--source-commit", + _SOURCE_COMMIT, + "--inventory", + "/home/private/release-inventory.json", + "--sdist", + "/home/private/netveil_audit-0.3.0.tar.gz", + "/home/private/secret.whl", + ] + ) + + self.assertEqual(result, 1) + self.assertEqual(stdout.getvalue(), "") + self.assertEqual( + stderr.getvalue(), + "netveil-wheel-verifier: wheel_path_invalid\n", + ) + self.assertNotIn("/home/private", stderr.getvalue()) + + def test_usage_is_stable(self) -> None: + stderr = io.StringIO() + + with redirect_stderr(stderr): + result = verifier.main([]) + + self.assertEqual(result, 2) + self.assertEqual( + stderr.getvalue(), + "netveil-wheel-verifier: usage_error\n", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_launcher.py b/tests/test_launcher.py new file mode 100644 index 0000000..eecdb28 --- /dev/null +++ b/tests/test_launcher.py @@ -0,0 +1,568 @@ +from __future__ import annotations + +import base64 +import contextlib +import hashlib +import io +import os +import stat +import sys +import tempfile +import unittest +from collections.abc import Iterator +from importlib import metadata +from pathlib import Path +from types import ModuleType +from typing import cast +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[1] +LAUNCHER_PATH = ROOT / "scripts" / "netveil-audit" +BOOTSTRAP_FIXTURE = ROOT / "tests" / "fixtures" / "launcher_bootstrap_fixture.py" + + +class _FakeHash: + def __init__(self, value: str, mode: str = "sha256") -> None: + self.mode = mode + self.value = value + + +class _FakeRecord: + def __init__( + self, + payload: bytes, + *, + path: str = "artifact", + mode: str = "sha256", + size: int | None = None, + ) -> None: + self.path = path + digest = hashlib.sha256(payload).digest() + value = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + self.hash: _FakeHash | None = _FakeHash(value, mode) + self.size = len(payload) if size is None else size + + def __str__(self) -> str: + return self.path + + +class _FakeDistribution: + def __init__( + self, + root: Path, + files: list[_FakeRecord] | None, + ) -> None: + self.root = root + self.files = files + self.metadata = { + "Name": "netveil-audit", + "Version": "0.3.0", + } + self.locations: dict[str, Path] = {} + + def locate_file(self, record: object) -> Path: + raw_path = str(record) + return self.locations.get(raw_path, self.root / raw_path) + + +def _load_launcher() -> ModuleType: + module = ModuleType("netveil_launcher_test") + module.__file__ = str(LAUNCHER_PATH) + source = LAUNCHER_PATH.read_bytes() + exec( # noqa: S102 - test loads the repository-owned launcher source. + compile(source, str(LAUNCHER_PATH), "exec", dont_inherit=True), + module.__dict__, + ) + return module + + +@contextlib.contextmanager +def _without_netveil_modules() -> Iterator[None]: + saved = { + name: module + for name, module in sys.modules.items() + if name == "netveil_bootstrap" + or name == "netveil" + or name.startswith("netveil.") + } + for name in saved: + sys.modules.pop(name, None) + try: + yield + finally: + for name in tuple(sys.modules): + if ( + name == "netveil_bootstrap" + or name == "netveil" + or name.startswith("netveil.") + ): + sys.modules.pop(name, None) + sys.modules.update(saved) + + +class LauncherBoundaryTests(unittest.TestCase): + def setUp(self) -> None: + self.launcher = _load_launcher() + self.bootstrap_payload = BOOTSTRAP_FIXTURE.read_bytes() + + def test_polyglot_header_requests_exact_isolation_profile(self) -> None: + lines = LAUNCHER_PATH.read_text().splitlines() + self.assertEqual(lines[0], "#!/bin/sh") + self.assertEqual( + lines[6], + 'exec "$netveil_script_directory/python" -IESB "$0" "$@"', + ) + self.assertFalse(self.launcher._startup_is_isolated()) + + def test_python_main_guard_fails_closed_without_isolation(self) -> None: + namespace = { + "__file__": str(LAUNCHER_PATH), + "__name__": "__main__", + } + stderr = io.StringIO() + with ( + patch.object(sys, "stderr", stderr), + self.assertRaises(SystemExit) as raised, + ): + exec( # noqa: S102 - test executes repository-owned launcher source. + compile( + LAUNCHER_PATH.read_bytes(), + str(LAUNCHER_PATH), + "exec", + dont_inherit=True, + ), + namespace, + ) + self.assertEqual(raised.exception.code, 10) + self.assertEqual(stderr.getvalue(), "netveil-audit: artifact_unverified\n") + + def test_site_root_is_bound_to_installed_prefix_layout(self) -> None: + with tempfile.TemporaryDirectory() as directory: + prefix = Path(directory) + launcher = prefix / "bin" / "netveil-audit" + launcher.parent.mkdir() + launcher.write_bytes(b"launcher") + site_root = ( + prefix + / "lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "site-packages" + ) + (site_root / "netveil_audit-0.3.0.dist-info").mkdir(parents=True) + self.assertEqual(self.launcher._site_root(launcher), site_root) + + duplicate = ( + prefix + / "lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "dist-packages" + / "netveil_audit-0.3.0.dist-info" + ) + duplicate.mkdir(parents=True) + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._site_root(launcher) + + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._site_root(Path("/tmp/netveil-audit")) + + def test_record_reader_pins_exact_regular_bytes(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "artifact" + payload = b"record-bound" + path.write_bytes(payload) + record = _FakeRecord(payload) + observed, identity = self.launcher._read_record_bound(path, record) + self.assertEqual(observed, payload) + self.assertEqual(identity, self.launcher._identity(path.stat())) + + path.write_bytes(b"record-b0und") + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_record_bound(path, record) + + def test_record_reader_rejects_contract_and_file_shape_failures(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + path = root / "artifact" + path.write_bytes(b"x") + for record in ( + _FakeRecord(b"x", mode="sha512"), + _FakeRecord(b"x", size=True), + _FakeRecord(b"x", size=-1), + _FakeRecord( + b"x", + size=self.launcher._MAX_ARTIFACT_FILE_BYTES + 1, + ), + ): + with ( + self.subTest(record=record), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + + no_hash = _FakeRecord(b"x") + no_hash.hash = None + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_record_bound(path, no_hash) + + directory_path = root / "directory" + directory_path.mkdir() + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_record_bound( + directory_path, + _FakeRecord(b"", size=0), + ) + + symlink = root / "link" + symlink.symlink_to(path) + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_record_bound(symlink, _FakeRecord(b"x")) + + oversized = root / "oversized" + oversized.write_bytes(b"x" * (self.launcher._MAX_ARTIFACT_FILE_BYTES + 1)) + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._read_bounded_regular(oversized) + + def test_record_reader_handles_io_and_close_failures(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "artifact" + path.write_bytes(b"x") + record = _FakeRecord(b"x") + with ( + patch.object(self.launcher.os, "open", side_effect=OSError), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + with ( + patch.object(self.launcher.os, "read", side_effect=OSError), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + with ( + patch.object(self.launcher.os, "close", side_effect=OSError), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + with ( + patch.object(self.launcher.os, "read", return_value=b"xx"), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._read_record_bound(path, record) + + def _installed_artifact( + self, + root: Path, + ) -> tuple[Path, Path, _FakeDistribution, bytes]: + prefix = root / "prefix" + launcher_path = prefix / "bin" / "netveil-audit" + launcher_path.parent.mkdir(parents=True) + launcher_payload = b"trusted launcher" + launcher_path.write_bytes(launcher_payload) + site_root = ( + prefix + / "lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "site-packages" + ) + dist_info = site_root / "netveil_audit-0.3.0.dist-info" + dist_info.mkdir(parents=True) + (dist_info / "METADATA").write_bytes(b"bounded metadata") + (dist_info / "RECORD").write_bytes(b"bounded record") + bootstrap_payload = b"trusted bootstrap" + (site_root / "netveil_bootstrap.py").write_bytes(bootstrap_payload) + launcher_record_path = os.path.relpath(launcher_path, site_root) + records = [ + _FakeRecord(launcher_payload, path=launcher_record_path), + _FakeRecord(bootstrap_payload, path="netveil_bootstrap.py"), + ] + return ( + launcher_path, + site_root, + _FakeDistribution(site_root, records), + bootstrap_payload, + ) + + def test_verified_bootstrap_binds_launcher_and_source_records(self) -> None: + with tempfile.TemporaryDirectory() as directory: + launcher_path, site_root, distribution, expected = self._installed_artifact( + Path(directory) + ) + original_path = list(sys.path) + try: + with patch.object( + self.launcher.importlib.metadata, + "PathDistribution", + return_value=cast(metadata.Distribution, distribution), + ): + payload, bootstrap_path = self.launcher._verified_bootstrap( + launcher_path + ) + finally: + sys.path[:] = original_path + self.assertEqual(payload, expected) + self.assertEqual(bootstrap_path, site_root / "netveil_bootstrap.py") + + def test_verified_bootstrap_rejects_metadata_and_record_drift(self) -> None: + mutations = ( + "missing-metadata", + "wrong-name", + "wrong-version", + "missing-launcher", + "missing-bootstrap", + "entry-points", + ) + for mutation in mutations: + with ( + self.subTest(mutation=mutation), + tempfile.TemporaryDirectory() as directory, + ): + launcher_path, _, distribution, _ = self._installed_artifact( + Path(directory) + ) + assert distribution.files is not None + if mutation == "missing-metadata": + distribution.metadata.pop("Name") + elif mutation == "wrong-name": + distribution.metadata["Name"] = "other" + elif mutation == "wrong-version": + distribution.metadata["Version"] = "9.9.9" + elif mutation == "missing-launcher": + distribution.files = [ + record + for record in distribution.files + if str(record) == "netveil_bootstrap.py" + ] + elif mutation == "missing-bootstrap": + distribution.files = [ + record + for record in distribution.files + if str(record) != "netveil_bootstrap.py" + ] + else: + site_root = distribution.root + ( + site_root / "netveil_audit-0.3.0.dist-info" / "entry_points.txt" + ).write_bytes(b"[console_scripts]\n") + with ( + patch.object( + self.launcher.importlib.metadata, + "PathDistribution", + return_value=cast(metadata.Distribution, distribution), + ), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._verified_bootstrap(launcher_path) + + def test_record_inventory_and_terminal_identity_fail_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + _, _, distribution, _ = self._installed_artifact(Path(directory)) + distribution.files = None + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._records(cast(metadata.Distribution, distribution)) + + duplicate = _FakeRecord(b"x", path="same") + distribution = _FakeDistribution(Path("/unused"), [duplicate, duplicate]) + with self.assertRaises(self.launcher._LaunchFailure): + self.launcher._records(cast(metadata.Distribution, distribution)) + + with tempfile.TemporaryDirectory() as directory: + launcher_path, _, distribution, _ = self._installed_artifact( + Path(directory) + ) + with ( + patch.object( + self.launcher.os, + "lstat", + side_effect=PermissionError, + ), + patch.object( + self.launcher.importlib.metadata, + "PathDistribution", + return_value=cast(metadata.Distribution, distribution), + ), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._verified_bootstrap(launcher_path) + + for mismatch_call in (2, 4): + with ( + self.subTest(mismatch_call=mismatch_call), + tempfile.TemporaryDirectory() as directory, + ): + launcher_path, _, distribution, _ = self._installed_artifact( + Path(directory) + ) + reads = [ + (b"launcher", (1,)), + (b"launcher", (1 if mismatch_call != 2 else 2,)), + (b"bootstrap", (3,)), + (b"bootstrap", (3 if mismatch_call != 4 else 4,)), + ] + with ( + patch.object( + self.launcher.importlib.metadata, + "PathDistribution", + return_value=cast(metadata.Distribution, distribution), + ), + patch.object( + self.launcher, + "_read_record_bound", + side_effect=reads, + ), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._verified_bootstrap(launcher_path) + + def test_verified_bootstrap_bytes_are_executed_in_memory(self) -> None: + with ( + _without_netveil_modules(), + patch.object(sys, "argv", ["netveil-audit", "safe"]), + ): + self.assertEqual( + self.launcher._execute_bootstrap( + self.bootstrap_payload, + BOOTSTRAP_FIXTURE, + ), + 23, + ) + + def test_preload_invalid_result_and_exception_fail_closed(self) -> None: + with ( + patch.dict(sys.modules, {"netveil": ModuleType("netveil")}), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._execute_bootstrap( + self.bootstrap_payload, + BOOTSTRAP_FIXTURE, + ) + + with ( + _without_netveil_modules(), + patch.object( + sys, + "argv", + ["netveil-audit", "invalid"], + ), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._execute_bootstrap(self.bootstrap_payload, BOOTSTRAP_FIXTURE) + + with _without_netveil_modules(): + with ( + patch.object(sys, "argv", ["netveil-audit", "explode"]), + self.assertRaises(RuntimeError), + ): + self.launcher._execute_bootstrap( + self.bootstrap_payload, + BOOTSTRAP_FIXTURE, + ) + self.assertNotIn("netveil_bootstrap", sys.modules) + + marker = "PRIVATE-LAUNCHER-MARKER" + stderr = io.StringIO() + with ( + patch.object( + self.launcher, + "_startup_is_isolated", + side_effect=RuntimeError(marker), + ), + patch.object(sys, "stderr", stderr), + ): + self.assertEqual( + self.launcher.main(), + self.launcher._INTERNAL_FAILURE_EXIT, + ) + self.assertEqual(stderr.getvalue(), "netveil-audit: internal_error\n") + self.assertNotIn(marker, stderr.getvalue()) + + def test_exact_error_writer_handles_partial_and_broken_output(self) -> None: + class PartialWriter(io.StringIO): + def write(self, value: str) -> int: + return super().write(value[:1]) + + writer = PartialWriter() + with patch.object(sys, "stderr", writer): + self.assertEqual( + self.launcher._write_failure("safe", 23), + 23, + ) + self.assertEqual(writer.getvalue(), "netveil-audit: safe\n") + + for written in (0, None, 100): + broken = io.StringIO() + with ( + patch.object(broken, "write", return_value=written), + patch.object(sys, "stderr", broken), + ): + self.assertEqual( + self.launcher._write_failure("safe", 23), + self.launcher._OUTPUT_FAILURE_EXIT, + ) + with patch.object(sys.stderr, "write", side_effect=OSError): + self.assertEqual( + self.launcher._write_failure("safe", 23), + self.launcher._OUTPUT_FAILURE_EXIT, + ) + + def test_file_flags_and_identity_are_fail_closed(self) -> None: + required = os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + self.assertEqual(self.launcher._file_flags() & required, required) + status = LAUNCHER_PATH.stat() + self.assertTrue(stat.S_ISREG(status.st_mode)) + self.assertEqual( + self.launcher._identity(status), + ( + status.st_dev, + status.st_ino, + status.st_mode, + status.st_uid, + status.st_gid, + status.st_nlink, + status.st_size, + status.st_mtime_ns, + status.st_ctime_ns, + ), + ) + with ( + patch("builtins.hasattr", return_value=False), + self.assertRaises(self.launcher._LaunchFailure), + ): + self.launcher._file_flags() + + def test_main_maps_success_and_failures_to_stable_codes(self) -> None: + stderr = io.StringIO() + with ( + patch.object(sys, "stderr", stderr), + patch.object(self.launcher, "_startup_is_isolated", return_value=False), + ): + self.assertEqual( + self.launcher.main(), + self.launcher._ARTIFACT_FAILURE_EXIT, + ) + self.assertEqual(stderr.getvalue(), "netveil-audit: artifact_unverified\n") + + with ( + patch.object(self.launcher, "_startup_is_isolated", return_value=True), + patch.object( + self.launcher, + "_verified_bootstrap", + return_value=(b"source", Path("/bootstrap")), + ), + patch.object(self.launcher, "_execute_bootstrap", return_value=23), + ): + self.assertEqual(self.launcher.main(), 23) + + stderr = io.StringIO() + with ( + patch.object(sys, "stderr", stderr), + patch.object(self.launcher, "_startup_is_isolated", return_value=True), + patch.object( + self.launcher, + "_verified_bootstrap", + side_effect=KeyboardInterrupt, + ), + ): + self.assertEqual( + self.launcher.main(), + self.launcher._INTERNAL_FAILURE_EXIT, + ) + self.assertEqual(stderr.getvalue(), "netveil-audit: interrupted\n") diff --git a/tests/test_model.py b/tests/test_model.py new file mode 100644 index 0000000..3314f68 --- /dev/null +++ b/tests/test_model.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import hashlib +import ipaddress +import unittest +from typing import cast + +from netveil import Endpoint, EndpointCorpus, EndpointScope, IPVersion +from netveil.model import _create_corpus, _create_endpoint + + +class ModelConstructionBoundaryTests(unittest.TestCase): + def test_public_constructors_are_blocked(self) -> None: + with self.assertRaisesRegex( + TypeError, + "^Endpoint objects are created by parse_corpus$", + ): + Endpoint() + with self.assertRaisesRegex( + TypeError, + "^EndpointCorpus objects are created by parse_corpus$", + ): + EndpointCorpus() + + with self.assertRaises(TypeError): + Endpoint( # type: ignore[call-arg] + "not-an-ip", + 0, + IPVersion.IPV4, + EndpointScope.DOCUMENTATION, + ) + with self.assertRaises(TypeError): + EndpointCorpus( # type: ignore[call-arg] + "not-a-sha256", + -1, + -1, + (), + (), + ) + + def test_endpoint_factory_enforces_exact_types_and_domains(self) -> None: + address = ipaddress.IPv4Address("192.0.2.1") + + with self.assertRaisesRegex(TypeError, "^address must be an exact"): + _create_endpoint("192.0.2.1", port=443) # type: ignore[arg-type] + with self.assertRaisesRegex(TypeError, "^port must be an exact int$"): + _create_endpoint(address, port=True) + for port in (0, 65_536): + with ( + self.subTest(port=port), + self.assertRaisesRegex(ValueError, "^port must be in"), + ): + _create_endpoint(address, port=port) + with self.assertRaisesRegex(ValueError, "^scoped IPv6"): + _create_endpoint( + ipaddress.IPv6Address("fe80::1%eth0"), + port=443, + ) + + def test_corpus_factory_derives_identity_counts_and_uniqueness(self) -> None: + payload = b"192.0.2.1:443\n192.0.2.1:443\n" + endpoint = _create_endpoint( + ipaddress.IPv4Address("192.0.2.1"), + port=443, + ) + corpus = _create_corpus( + payload=payload, + physical_line_count=2, + endpoints=(endpoint, endpoint), + ) + + self.assertEqual(corpus.source_sha256, hashlib.sha256(payload).hexdigest()) + self.assertEqual(corpus.source_bytes, len(payload)) + self.assertEqual(corpus.physical_line_count, 2) + self.assertEqual(corpus.endpoints, (endpoint, endpoint)) + self.assertEqual(corpus.unique_endpoints, (endpoint,)) + self.assertEqual(corpus.duplicate_count, 1) + + def test_corpus_factory_rejects_impossible_inputs(self) -> None: + endpoint = _create_endpoint( + ipaddress.IPv4Address("192.0.2.1"), + port=443, + ) + + with self.assertRaisesRegex(TypeError, "^payload must be exact bytes$"): + _create_corpus( + payload=cast(bytes, bytearray(b"x")), + physical_line_count=1, + endpoints=(endpoint,), + ) + with self.assertRaisesRegex( + TypeError, + "^physical_line_count must be an exact int$", + ): + _create_corpus( + payload=b"x", + physical_line_count=True, + endpoints=(endpoint,), + ) + with self.assertRaisesRegex(TypeError, "^endpoints must be an exact tuple$"): + _create_corpus( + payload=b"x", + physical_line_count=1, + endpoints=cast(tuple[Endpoint, ...], [endpoint]), + ) + with self.assertRaisesRegex(ValueError, "^physical_line_count must be"): + _create_corpus( + payload=b"x", + physical_line_count=-1, + endpoints=(endpoint,), + ) + with self.assertRaisesRegex(ValueError, "^a corpus must contain"): + _create_corpus( + payload=b"", + physical_line_count=0, + endpoints=(), + ) + with self.assertRaisesRegex(ValueError, "^physical_line_count cannot"): + _create_corpus( + payload=b"x", + physical_line_count=1, + endpoints=(endpoint, endpoint), + ) + with self.assertRaisesRegex(TypeError, "^every endpoint must come"): + _create_corpus( + payload=b"x", + physical_line_count=1, + endpoints=cast(tuple[Endpoint, ...], (object(),)), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_packaging.py b/tests/test_packaging.py new file mode 100644 index 0000000..7ad3b72 --- /dev/null +++ b/tests/test_packaging.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import stat +import tomllib +import unittest +from importlib import resources +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +class PackagingContractTests(unittest.TestCase): + def test_pep561_marker_is_packaged(self) -> None: + marker = resources.files("netveil").joinpath("py.typed") + self.assertTrue(marker.is_file()) + self.assertIn(marker.read_bytes(), (b"", b"\n")) + + def test_static_launcher_replaces_generated_entry_point(self) -> None: + document = tomllib.loads((ROOT / "pyproject.toml").read_text()) + project = document["project"] + setuptools = document["tool"]["setuptools"] + self.assertNotIn("scripts", project) + self.assertEqual(project["dependencies"], []) + self.assertEqual(setuptools["script-files"], ["scripts/netveil-audit"]) + self.assertEqual(setuptools["py-modules"], ["netveil_bootstrap"]) + + launcher = ROOT / "scripts" / "netveil-audit" + self.assertEqual(launcher.read_bytes().splitlines()[0], b"#!/bin/sh") + self.assertTrue(launcher.stat().st_mode & stat.S_IXUSR) + + def test_source_manifest_keeps_release_evidence_reproducible(self) -> None: + manifest = (ROOT / "MANIFEST.in").read_text().splitlines() + required = { + "include README.md", + "include SECURITY.md", + "include requirements-dev.txt", + "include scripts/netveil-audit", + "recursive-include docs *.md", + "recursive-include docs/assets *.svg *.json *.cast *.gif", + "recursive-include tests *.py", + "recursive-include tools *.py", + } + self.assertTrue(required.issubset(manifest)) + + def test_developer_tool_versions_are_exactly_pinned(self) -> None: + requirements = (ROOT / "requirements-dev.txt").read_text().splitlines() + pins = [line for line in requirements if line and not line.startswith("#")] + self.assertGreaterEqual(len(pins), 4) + self.assertTrue(all(line.count("==") == 1 for line in pins)) + + def test_package_metadata_identifies_omar_and_public_project_links(self) -> None: + document = tomllib.loads((ROOT / "pyproject.toml").read_text()) + project = document["project"] + + self.assertEqual(project["authors"], [{"name": "Omar Ibrahim"}]) + self.assertEqual( + project["urls"]["Repository"], + "https://github.com/omar07ibrahim/Hello-World", + ) + self.assertIn("Typing :: Typed", project["classifiers"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_parser.py b/tests/test_parser.py new file mode 100644 index 0000000..fe532e2 --- /dev/null +++ b/tests/test_parser.py @@ -0,0 +1,298 @@ +from __future__ import annotations + +import hashlib +import ipaddress +import socket +import traceback +import unittest +from unittest.mock import patch + +import netveil.model as model_module +from netveil import ( + MAX_INPUT_BYTES, + MAX_PHYSICAL_LINES, + EndpointParseError, + EndpointParseErrorCode, + EndpointScope, + IPVersion, + parse_corpus, +) + + +class SyntheticGlobalIPv4(ipaddress.IPv4Address): + """Exercise the global branch without committing a live endpoint.""" + + @property + def is_unspecified(self) -> bool: + return False + + @property + def is_loopback(self) -> bool: + return False + + @property + def is_multicast(self) -> bool: + return False + + @property + def is_link_local(self) -> bool: + return False + + @property + def is_reserved(self) -> bool: + return False + + @property + def is_private(self) -> bool: + return False + + +class DecodeOverridingBytes(bytes): + def decode( + self, + encoding: str = "utf-8", + errors: str = "strict", + ) -> str: + del encoding, errors + return "192.0.2.1:443\n" + + +class EndpointParserHappyPathTests(unittest.TestCase): + def test_parses_canonical_ipv4_and_ipv6_without_network_access(self) -> None: + payload = ( + b"# Synthetic IETF documentation ranges only\n" + b"192.0.2.10:443\n" + b"[2001:0DB8:0:0:0:0:0:10]:8443\n" + ) + with ( + patch.object(socket, "socket", side_effect=AssertionError("network")), + patch.object( + socket, + "getaddrinfo", + side_effect=AssertionError("resolution"), + ), + ): + corpus = parse_corpus(payload) + + self.assertEqual(corpus.source_sha256, hashlib.sha256(payload).hexdigest()) + self.assertEqual(corpus.source_bytes, len(payload)) + self.assertEqual(corpus.physical_line_count, 3) + self.assertEqual(corpus.endpoint_count, 2) + self.assertEqual(corpus.unique_count, 2) + self.assertEqual(corpus.duplicate_count, 0) + self.assertEqual(corpus.endpoints[0].canonical, "192.0.2.10:443") + self.assertEqual(corpus.endpoints[0].version, IPVersion.IPV4) + self.assertEqual(corpus.endpoints[0].scope, EndpointScope.DOCUMENTATION) + self.assertEqual(corpus.endpoints[1].canonical, "[2001:db8::10]:8443") + self.assertEqual(corpus.endpoints[1].version, IPVersion.IPV6) + self.assertEqual(corpus.endpoints[1].scope, EndpointScope.DOCUMENTATION) + + def test_accepts_crlf_and_preserves_exact_source_hash(self) -> None: + payload = b"198.51.100.8:80\r\n203.0.113.9:443\r\n" + corpus = parse_corpus(payload) + self.assertEqual(corpus.physical_line_count, 2) + self.assertEqual(corpus.source_sha256, hashlib.sha256(payload).hexdigest()) + + without_final_newline = parse_corpus(b"192.0.2.10:443") + self.assertEqual(without_final_newline.physical_line_count, 1) + + def test_equivalent_ipv6_spellings_are_duplicates(self) -> None: + corpus = parse_corpus(b"[2001:db8::1]:443\n[2001:0DB8:0:0:0:0:0:1]:443\n") + self.assertEqual(corpus.endpoint_count, 2) + self.assertEqual(corpus.unique_count, 1) + self.assertEqual(corpus.duplicate_count, 1) + self.assertEqual(corpus.unique_endpoints[0].canonical, "[2001:db8::1]:443") + + def test_classifies_privacy_relevant_scopes(self) -> None: + corpus = parse_corpus( + b"10.0.0.1:1\n" + b"127.0.0.1:2\n" + b"169.254.1.1:3\n" + b"224.0.0.1:4\n" + b"0.0.0.0:5\n" + b"100.64.0.1:6\n" + b"240.0.0.1:7\n" + b"[fec0::1]:8\n" + ) + self.assertEqual( + [endpoint.scope for endpoint in corpus.endpoints], + [ + EndpointScope.PRIVATE, + EndpointScope.LOOPBACK, + EndpointScope.LINK_LOCAL, + EndpointScope.MULTICAST, + EndpointScope.UNSPECIFIED, + EndpointScope.SHARED, + EndpointScope.RESERVED, + EndpointScope.SITE_LOCAL, + ], + ) + + def test_global_scope_branch_uses_no_live_endpoint_fixture(self) -> None: + address = SyntheticGlobalIPv4("192.0.2.1") + with patch.object(model_module, "_DOCUMENTATION_NETWORKS", ()): + self.assertEqual( + model_module._scope(address), + EndpointScope.GLOBAL, + ) + + +class EndpointParserFailureTests(unittest.TestCase): + def assert_rejected( + self, + payload: bytes, + code: EndpointParseErrorCode, + *, + line_number: int | None, + forbidden: str | None = None, + ) -> None: + with self.assertRaises(EndpointParseError) as raised: + parse_corpus(payload) + self.assertEqual(raised.exception.code, code) + self.assertEqual(raised.exception.line_number, line_number) + if forbidden is not None: + self.assertNotIn(forbidden, str(raised.exception)) + + def test_rejects_empty_or_comment_only_corpora(self) -> None: + for payload in (b"", b"\n", b"# no endpoints\n"): + with self.subTest(payload=payload): + self.assert_rejected( + payload, + EndpointParseErrorCode.EMPTY_CORPUS, + line_number=None, + ) + + def test_rejects_ambiguous_or_invalid_syntax(self) -> None: + cases = ( + (b" 192.0.2.1:443\n", "192.0.2.1"), + (b"192.0.2.1:443 # inline\n", "192.0.2.1"), + (b"2001:db8::1:443\n", "2001:db8"), + (b"[192.0.2.1]:443\n", "192.0.2.1"), + ) + for payload, forbidden in cases: + with self.subTest(payload=payload): + self.assert_rejected( + payload, + EndpointParseErrorCode.INVALID_SYNTAX, + line_number=1, + forbidden=forbidden, + ) + self.assert_rejected( + b"example.invalid:443\n", + EndpointParseErrorCode.INVALID_ADDRESS, + line_number=1, + forbidden="example.invalid", + ) + + def test_rejects_invalid_addresses_without_echoing_them(self) -> None: + for payload, forbidden in ( + (b"999.51.100.7:443\n", "999.51.100.7"), + (b"[fe80::1%eth0]:443\n", "eth0"), + ): + with self.subTest(payload=payload): + self.assert_rejected( + payload, + EndpointParseErrorCode.INVALID_ADDRESS, + line_number=1, + forbidden=forbidden, + ) + + def test_invalid_address_creates_no_library_exception_context(self) -> None: + payload = b"PRIVATE-ENDPOINT-VALUE:443\n" + + with self.assertRaises(EndpointParseError) as raised: + parse_corpus(payload) + + error = raised.exception + rendered = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + self.assertEqual(error.code, EndpointParseErrorCode.INVALID_ADDRESS) + self.assertIsNone(error.__context__) + self.assertIsNone(error.__cause__) + self.assertNotIn("PRIVATE-ENDPOINT-VALUE", repr(error)) + self.assertNotIn("PRIVATE-ENDPOINT-VALUE", rendered) + self.assertNotIn("AddressValueError", rendered) + + def test_rejects_invalid_and_noncanonical_ports(self) -> None: + cases = ( + (b"192.0.2.1:0\n", EndpointParseErrorCode.INVALID_PORT), + (b"192.0.2.1:65536\n", EndpointParseErrorCode.INVALID_PORT), + (b"192.0.2.1:100000\n", EndpointParseErrorCode.INVALID_PORT), + (b"192.0.2.1:0443\n", EndpointParseErrorCode.NON_CANONICAL_PORT), + ) + for payload, code in cases: + with self.subTest(payload=payload): + self.assert_rejected(payload, code, line_number=1) + + def test_rejects_invalid_utf8_and_non_lf_line_endings(self) -> None: + self.assert_rejected( + b"\xff:443\n", + EndpointParseErrorCode.INVALID_UTF8, + line_number=None, + ) + self.assert_rejected( + b"192.0.2.1:443\r192.0.2.2:443\n", + EndpointParseErrorCode.INVALID_LINE_ENDING, + line_number=None, + ) + for separator in ( + "\u000b", + "\u000c", + "\u001c", + "\u001d", + "\u001e", + "\u0085", + "\u2028", + "\u2029", + ): + with self.subTest(separator=separator.encode().hex()): + self.assert_rejected( + f"192.0.2.1:443{separator}192.0.2.2:443\n".encode(), + EndpointParseErrorCode.INVALID_LINE_ENDING, + line_number=None, + ) + self.assert_rejected( + f"# comment{separator}hidden line\n192.0.2.1:443\n".encode(), + EndpointParseErrorCode.INVALID_LINE_ENDING, + line_number=None, + ) + + def test_invalid_utf8_creates_no_library_exception_context(self) -> None: + payload = b"PRIVATE-CORPUS-\xff-DO-NOT-ECHO" + + with self.assertRaises(EndpointParseError) as raised: + parse_corpus(payload) + + error = raised.exception + rendered = "".join( + traceback.format_exception(type(error), error, error.__traceback__) + ) + self.assertEqual(error.code, EndpointParseErrorCode.INVALID_UTF8) + self.assertIsNone(error.__context__) + self.assertIsNone(error.__cause__) + self.assertNotIn(repr(payload), repr(error)) + self.assertNotIn("PRIVATE-CORPUS", rendered) + self.assertNotIn("UnicodeDecodeError", rendered) + + def test_enforces_resource_bounds(self) -> None: + self.assert_rejected( + b"x" * (MAX_INPUT_BYTES + 1), + EndpointParseErrorCode.INPUT_TOO_LARGE, + line_number=None, + ) + self.assert_rejected( + b"\n" * (MAX_PHYSICAL_LINES + 1), + EndpointParseErrorCode.TOO_MANY_LINES, + line_number=None, + ) + + def test_requires_exact_bytes_type(self) -> None: + with self.assertRaisesRegex(TypeError, "^payload must be exact bytes$"): + parse_corpus("192.0.2.1:443\n") # type: ignore[arg-type] + with self.assertRaisesRegex(TypeError, "^payload must be exact bytes$"): + parse_corpus(DecodeOverridingBytes(b"not an endpoint")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_privacy.py b/tests/test_privacy.py new file mode 100644 index 0000000..bcf7fed --- /dev/null +++ b/tests/test_privacy.py @@ -0,0 +1,597 @@ +from __future__ import annotations + +import hashlib +import json +import socket +import subprocess +import sys +import unittest +from dataclasses import FrozenInstanceError +from typing import Any, cast +from unittest.mock import patch + +from netveil import ( + CANONICAL_JSON_PROTOCOL, + MAX_INPUT_BYTES, + MAX_PHYSICAL_LINES, + MIN_PSEUDONYMIZATION_KEY_BYTES, + PRIVACY_RECEIPT_SCHEMA, + PRIVACY_REPORT_SCHEMA, + PSEUDONYMIZATION_PROTOCOL, + RUNTIME_PROFILE_SCHEMA, + DuplicateGroup, + EndpointParseError, + EndpointParseErrorCode, + PrivacyReceipt, + PrivacyReport, + RuntimeProfile, + build_privacy_receipt, + build_privacy_report, +) +from netveil.privacy import ( + _DUPLICATE_GROUP_DOMAIN, + _SOURCE_CONTENT_DOMAIN, + _create_duplicate_group, + _create_privacy_receipt, + _create_privacy_report, + _create_runtime_profile, + _keyed_identifier, +) + +_KEY_A = bytes(range(32)) +_KEY_B = bytes(range(32, 64)) +_PAYLOAD = ( + b"# Synthetic special-use ranges only\n" + b"192.0.2.1:80\n" + b"192.0.2.1:80\n" + b"[2001:db8::1]:443\n" + b"[2001:0DB8:0:0:0:0:0:1]:443\n" + b"10.0.0.1:1024\n" + b"127.0.0.1:49152\n" +) +_RAW_VALUES = ( + "192.0.2.1:80", + "[2001:db8::1]:443", + "[2001:0DB8:0:0:0:0:0:1]:443", + "10.0.0.1:1024", + "127.0.0.1:49152", +) + + +def _json(payload: bytes) -> dict[str, Any]: + document = json.loads(payload) + if not isinstance(document, dict): + raise TypeError("expected a JSON object") + return cast(dict[str, Any], document) + + +class PrivacyReportTests(unittest.TestCase): + def test_builds_expected_aggregates_without_network_or_process_calls(self) -> None: + with ( + patch.object(socket, "socket", side_effect=AssertionError("network")), + patch.object( + socket, + "getaddrinfo", + side_effect=AssertionError("resolution"), + ), + patch.object(subprocess, "Popen", side_effect=AssertionError("process")), + patch.object(subprocess, "run", side_effect=AssertionError("process")), + ): + report = build_privacy_report( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + + document = _json(report.canonical_json_bytes()) + self.assertEqual(document["schema"], PRIVACY_REPORT_SCHEMA) + self.assertEqual(document["protocol"], PSEUDONYMIZATION_PROTOCOL) + self.assertEqual(document["canonicalization"], CANONICAL_JSON_PROTOCOL) + self.assertEqual( + document["runtime"], + { + "endpoint_semantics": "python-stdlib-ipaddress", + "python_implementation": sys.implementation.name, + "python_version": ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + ), + "schema": RUNTIME_PROFILE_SCHEMA, + }, + ) + self.assertEqual( + document["counts"], + { + "endpoint_occurrences": 6, + "physical_lines": 7, + "source_bytes": len(_PAYLOAD), + "unique_endpoints": 4, + }, + ) + self.assertEqual( + document["endpoint_occurrences_by_ip_version"], + {"ipv4": 4, "ipv6": 2}, + ) + self.assertEqual( + document["endpoint_occurrences_by_port_bucket"], + { + "dynamic_49152_65535": 1, + "registered_1024_49151": 1, + "system_1_1023": 4, + }, + ) + scopes = document["endpoint_occurrences_by_scope"] + self.assertEqual(scopes["documentation"], 4) + self.assertEqual(scopes["private"], 1) + self.assertEqual(scopes["loopback"], 1) + self.assertEqual(sum(scopes.values()), 6) + self.assertEqual( + document["duplicates"]["extra_occurrences"], + 2, + ) + self.assertEqual(document["duplicates"]["group_count"], 2) + groups = document["duplicates"]["groups"] + self.assertEqual([group["occurrences"] for group in groups], [2, 2]) + self.assertEqual([group["extra_occurrences"] for group in groups], [1, 1]) + self.assertEqual( + [group["id"] for group in groups], + sorted(group["id"] for group in groups), + ) + + def test_output_and_failures_do_not_reveal_raw_values_or_source_sha(self) -> None: + receipt = build_privacy_receipt( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + rendered = ( + receipt.canonical_json_bytes().decode("ascii") + + repr(receipt) + + repr(receipt.report) + ) + for raw_value in _RAW_VALUES: + self.assertNotIn(raw_value, rendered) + self.assertNotIn(hashlib.sha256(_PAYLOAD).hexdigest(), rendered) + self.assertNotIn(_KEY_A.hex(), rendered) + + invalid_value = "invalid-sensitive-value" + with self.assertRaises(EndpointParseError) as raised: + build_privacy_report( + f"{invalid_value}:443\n".encode(), + pseudonymization_key=_KEY_A, + ) + self.assertEqual( + raised.exception.code, + EndpointParseErrorCode.INVALID_ADDRESS, + ) + self.assertNotIn(invalid_value, str(raised.exception)) + self.assertNotIn(invalid_value, repr(raised.exception)) + + def test_report_and_receipt_are_deterministic_canonical_json(self) -> None: + first = build_privacy_receipt( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + second = build_privacy_receipt( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + self.assertEqual(first, second) + self.assertEqual(first.canonical_json_bytes(), second.canonical_json_bytes()) + + report_bytes = first.report.canonical_json_bytes() + receipt_document = _json(first.canonical_json_bytes()) + self.assertEqual(receipt_document["schema"], PRIVACY_RECEIPT_SCHEMA) + self.assertEqual( + receipt_document["report"], + _json(report_bytes), + ) + self.assertEqual( + first.report_sha256, + hashlib.sha256(report_bytes).hexdigest(), + ) + self.assertEqual( + receipt_document["report_digest"], + {"algorithm": "sha256", "value": first.report_sha256}, + ) + self.assertNotIn("report_digest", _json(report_bytes)) + + for canonical_bytes in (report_bytes, first.canonical_json_bytes()): + self.assertFalse(canonical_bytes.endswith(b"\n")) + self.assertEqual( + canonical_bytes, + json.dumps( + json.loads(canonical_bytes), + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii"), + ) + + def test_key_rotation_and_domain_separation_change_typed_identifiers(self) -> None: + first = build_privacy_report( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + rotated = build_privacy_report( + _PAYLOAD, + pseudonymization_key=_KEY_B, + ) + self.assertNotEqual(first.source_content_id, rotated.source_content_id) + self.assertTrue(first.source_content_id.startswith("nvs1_")) + self.assertTrue( + all(group.group_id.startswith("nvd1_") for group in first.duplicate_groups) + ) + self.assertNotEqual( + {group.group_id for group in first.duplicate_groups}, + {group.group_id for group in rotated.duplicate_groups}, + ) + + same_value = b"same-framed-value" + source_id = _keyed_identifier( + _KEY_A, + domain=_SOURCE_CONTENT_DOMAIN, + value=same_value, + prefix="nvs1_", + ) + duplicate_id = _keyed_identifier( + _KEY_A, + domain=_DUPLICATE_GROUP_DOMAIN, + value=same_value, + prefix="nvd1_", + ) + self.assertEqual( + source_id, + "nvs1_1835ed652ea947a88125064fad24a25b4cd3faa2c4f3e05878ce2bfe8fdd66c5", + ) + self.assertEqual( + duplicate_id, + "nvd1_c5422fb8589253f084b16b227ba8fc7a8d1e2f69ec257f93a10f4a2544941750", + ) + self.assertNotEqual( + source_id.removeprefix("nvs1_"), duplicate_id.removeprefix("nvd1_") + ) + self.assertNotEqual(source_id, duplicate_id) + + def test_equivalent_endpoints_have_stable_duplicate_ids_across_corpora( + self, + ) -> None: + first = build_privacy_report( + b"[2001:db8::1]:443\n[2001:0DB8:0:0:0:0:0:1]:443\n", + pseudonymization_key=_KEY_A, + ) + second = build_privacy_report( + b"# another source\n[2001:db8::1]:443\n[2001:db8::1]:443\n", + pseudonymization_key=_KEY_A, + ) + self.assertNotEqual(first.source_content_id, second.source_content_id) + self.assertEqual( + first.duplicate_groups[0].group_id, + second.duplicate_groups[0].group_id, + ) + + def test_single_endpoint_report_has_no_duplicate_identifier(self) -> None: + report = build_privacy_report( + b"192.0.2.1:443\n", + pseudonymization_key=_KEY_A, + ) + self.assertEqual(report.duplicate_groups, ()) + self.assertEqual(report.duplicate_group_count, 0) + self.assertEqual(report.duplicate_occurrence_count, 0) + + def test_exact_types_key_length_and_parser_bounds_fail_closed(self) -> None: + class BytesSubclass(bytes): + pass + + for payload in ( + "192.0.2.1:443\n", + bytearray(b"192.0.2.1:443\n"), + BytesSubclass(b"192.0.2.1:443\n"), + ): + with ( + self.subTest(payload_type=type(payload).__name__), + self.assertRaisesRegex(TypeError, "^payload must be exact bytes$"), + ): + build_privacy_report( + payload, # type: ignore[arg-type] + pseudonymization_key=_KEY_A, + ) + + for key in ( + "x" * MIN_PSEUDONYMIZATION_KEY_BYTES, + bytearray(_KEY_A), + BytesSubclass(_KEY_A), + ): + with ( + self.subTest(key_type=type(key).__name__), + self.assertRaisesRegex( + TypeError, + "^pseudonymization_key must be exact bytes$", + ), + ): + build_privacy_report( + b"192.0.2.1:443\n", + pseudonymization_key=key, # type: ignore[arg-type] + ) + + short_key = b"sensitive-but-too-short" + with self.assertRaisesRegex( + ValueError, + "^pseudonymization_key must contain at least 32 bytes$", + ) as raised: + build_privacy_report( + b"192.0.2.1:443\n", + pseudonymization_key=short_key, + ) + self.assertNotIn(short_key.decode(), str(raised.exception)) + + with self.assertRaises(EndpointParseError) as oversized: + build_privacy_report( + b"x" * (MAX_INPUT_BYTES + 1), + pseudonymization_key=_KEY_A, + ) + self.assertEqual( + oversized.exception.code, + EndpointParseErrorCode.INPUT_TOO_LARGE, + ) + + def test_public_models_are_frozen_and_direct_construction_is_blocked(self) -> None: + for model, message in ( + ( + RuntimeProfile, + "RuntimeProfile objects are created by build_privacy_report", + ), + ( + DuplicateGroup, + "DuplicateGroup objects are created by build_privacy_report", + ), + ( + PrivacyReport, + "PrivacyReport objects are created by build_privacy_report", + ), + ( + PrivacyReceipt, + "PrivacyReceipt objects are created by build_privacy_receipt", + ), + ): + with ( + self.subTest(model=model.__name__), + self.assertRaisesRegex(TypeError, f"^{message}$"), + ): + model() + + receipt = build_privacy_receipt( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + with self.assertRaises(FrozenInstanceError): + receipt.report.endpoint_count = 0 # type: ignore[misc] + with self.assertRaises(FrozenInstanceError): + receipt.report.duplicate_groups[0].occurrences = 99 # type: ignore[misc] + with self.assertRaises(FrozenInstanceError): + receipt.report_sha256 = "0" * 64 # type: ignore[misc] + + +class PrivacyModelInvariantTests(unittest.TestCase): + def setUp(self) -> None: + self.report = build_privacy_report( + _PAYLOAD, + pseudonymization_key=_KEY_A, + ) + self.valid: dict[str, object] = { + "source_content_id": self.report.source_content_id, + "runtime_profile": self.report.runtime_profile, + "source_bytes": self.report.source_bytes, + "physical_line_count": self.report.physical_line_count, + "endpoint_count": self.report.endpoint_count, + "unique_endpoint_count": self.report.unique_endpoint_count, + "ip_version_counts": self.report.ip_version_counts, + "scope_counts": self.report.scope_counts, + "port_bucket_counts": self.report.port_bucket_counts, + "duplicate_groups": self.report.duplicate_groups, + } + + def create(self, **changes: object) -> PrivacyReport: + values = self.valid | changes + return _create_privacy_report( + source_content_id=cast(str, values["source_content_id"]), + runtime_profile=cast(RuntimeProfile, values["runtime_profile"]), + source_bytes=cast(int, values["source_bytes"]), + physical_line_count=cast(int, values["physical_line_count"]), + endpoint_count=cast(int, values["endpoint_count"]), + unique_endpoint_count=cast(int, values["unique_endpoint_count"]), + ip_version_counts=cast( + tuple[tuple[str, int], ...], + values["ip_version_counts"], + ), + scope_counts=cast( + tuple[tuple[str, int], ...], + values["scope_counts"], + ), + port_bucket_counts=cast( + tuple[tuple[str, int], ...], + values["port_bucket_counts"], + ), + duplicate_groups=cast( + tuple[DuplicateGroup, ...], + values["duplicate_groups"], + ), + ) + + def test_duplicate_group_factory_rejects_invalid_models(self) -> None: + with self.assertRaisesRegex(TypeError, "^group_id must be an exact str$"): + _create_duplicate_group(group_id=cast(str, b"x"), occurrences=2) + for group_id in ( + "wrong_" + "0" * 64, + "nvd1_short", + "nvd1_" + "G" * 64, + ): + with ( + self.subTest(group_id=group_id), + self.assertRaisesRegex(ValueError, "^group_id must be a valid"), + ): + _create_duplicate_group(group_id=group_id, occurrences=2) + with self.assertRaisesRegex(TypeError, "^occurrences must be an exact int$"): + _create_duplicate_group( + group_id="nvd1_" + "0" * 64, + occurrences=True, + ) + for occurrences in (-1, 0, 1): + with ( + self.subTest(occurrences=occurrences), + self.assertRaises(ValueError), + ): + _create_duplicate_group( + group_id="nvd1_" + "0" * 64, + occurrences=occurrences, + ) + + def test_report_factory_rejects_invalid_identifiers_and_scalar_counts( + self, + ) -> None: + with self.assertRaisesRegex(TypeError, "^source_content_id must be"): + self.create(source_content_id=cast(str, b"x")) + with self.assertRaisesRegex(ValueError, "^source_content_id must be"): + self.create(source_content_id="nvs1_" + "z" * 64) + with self.assertRaisesRegex(TypeError, "^runtime_profile must come"): + self.create(runtime_profile=object()) + + forged_profile: RuntimeProfile = object.__new__(RuntimeProfile) + object.__setattr__(forged_profile, "schema", RUNTIME_PROFILE_SCHEMA) + object.__setattr__( + forged_profile, + "python_implementation", + sys.implementation.name, + ) + object.__setattr__(forged_profile, "python_version", "0.0.0") + object.__setattr__( + forged_profile, + "endpoint_semantics", + "python-stdlib-ipaddress", + ) + with self.assertRaisesRegex(ValueError, "^runtime_profile does not match"): + self.create(runtime_profile=forged_profile) + + self.assertEqual(self.report.runtime_profile, _create_runtime_profile()) + + for field in ( + "source_bytes", + "physical_line_count", + "endpoint_count", + "unique_endpoint_count", + ): + with ( + self.subTest(field=field), + self.assertRaisesRegex(TypeError, f"^{field} must be an exact int$"), + ): + self.create(**{field: True}) + with self.assertRaisesRegex(ValueError, "^source_bytes must be non-negative$"): + self.create(source_bytes=-1) + with self.assertRaisesRegex(ValueError, "^source_bytes must be positive$"): + self.create(source_bytes=0) + with self.assertRaisesRegex(ValueError, "^source_bytes exceeds"): + self.create(source_bytes=MAX_INPUT_BYTES + 1) + with self.assertRaisesRegex(ValueError, "^physical_line_count exceeds"): + self.create(physical_line_count=MAX_PHYSICAL_LINES + 1) + with self.assertRaisesRegex(ValueError, "^endpoint_count must be positive$"): + self.create(endpoint_count=0, unique_endpoint_count=0) + for unique_count in (0, self.report.endpoint_count + 1): + with ( + self.subTest(unique_count=unique_count), + self.assertRaisesRegex(ValueError, "^unique_endpoint_count must be"), + ): + self.create(unique_endpoint_count=unique_count) + with self.assertRaisesRegex(ValueError, "^physical_line_count cannot"): + self.create(physical_line_count=self.report.endpoint_count - 1) + with self.assertRaisesRegex(ValueError, "^source_bytes is too small"): + self.create(source_bytes=1) + + def test_report_factory_rejects_invalid_count_tables(self) -> None: + with self.assertRaisesRegex(TypeError, "^ip_version_counts must be"): + self.create(ip_version_counts=list(self.report.ip_version_counts)) + with self.assertRaisesRegex( + TypeError, + "^ip_version_counts entries must be", + ): + self.create(ip_version_counts=(("ipv4", 4), ["ipv6", 2])) + with self.assertRaisesRegex( + TypeError, + "^ip_version_counts entries must be", + ): + self.create(ip_version_counts=(("ipv4", 4, 0), ("ipv6", 2))) + with self.assertRaisesRegex( + TypeError, + "^ip_version_counts labels must be", + ): + self.create(ip_version_counts=((cast(str, b"ipv4"), 4), ("ipv6", 2))) + with self.assertRaisesRegex( + TypeError, + "^ip_version_counts count must be an exact int$", + ): + self.create(ip_version_counts=(("ipv4", True), ("ipv6", 5))) + with self.assertRaisesRegex( + ValueError, + "^ip_version_counts count must be non-negative$", + ): + self.create(ip_version_counts=(("ipv4", -1), ("ipv6", 7))) + with self.assertRaisesRegex( + ValueError, + "^ip_version_counts labels or order are invalid$", + ): + self.create(ip_version_counts=(("ipv6", 2), ("ipv4", 4))) + with self.assertRaisesRegex( + ValueError, + "^ip_version_counts must sum to endpoint_count$", + ): + self.create(ip_version_counts=(("ipv4", 3), ("ipv6", 2))) + + def test_report_factory_rejects_invalid_duplicate_groups(self) -> None: + with self.assertRaisesRegex(TypeError, "^duplicate_groups must be"): + self.create(duplicate_groups=list(self.report.duplicate_groups)) + with self.assertRaisesRegex(TypeError, "^every duplicate group must"): + self.create(duplicate_groups=(object(),)) + + first, second = self.report.duplicate_groups + invalid_id: DuplicateGroup = object.__new__(DuplicateGroup) + object.__setattr__(invalid_id, "group_id", "invalid") + object.__setattr__(invalid_id, "occurrences", 2) + with self.assertRaisesRegex(ValueError, "^group_id must be a valid"): + self.create(duplicate_groups=(invalid_id,)) + + invalid_type: DuplicateGroup = object.__new__(DuplicateGroup) + object.__setattr__(invalid_type, "group_id", first.group_id) + object.__setattr__(invalid_type, "occurrences", True) + with self.assertRaisesRegex(TypeError, "^occurrences must be an exact int$"): + self.create(duplicate_groups=(invalid_type,)) + + too_small: DuplicateGroup = object.__new__(DuplicateGroup) + object.__setattr__(too_small, "group_id", first.group_id) + object.__setattr__(too_small, "occurrences", 1) + with self.assertRaisesRegex(ValueError, "^duplicate group occurrences"): + self.create(duplicate_groups=(too_small,)) + + with self.assertRaisesRegex(ValueError, "^duplicate_groups must have"): + self.create(duplicate_groups=(second, first)) + with self.assertRaisesRegex(ValueError, "^duplicate_groups must have"): + self.create(duplicate_groups=(first, first)) + + too_many_groups = ( + _create_duplicate_group(group_id="nvd1_" + "0" * 64, occurrences=2), + _create_duplicate_group(group_id="nvd1_" + "1" * 64, occurrences=2), + _create_duplicate_group(group_id="nvd1_" + "2" * 64, occurrences=3), + ) + with self.assertRaisesRegex(ValueError, "^duplicate_group_count cannot"): + self.create( + unique_endpoint_count=2, + duplicate_groups=too_many_groups, + ) + with self.assertRaisesRegex(ValueError, "^duplicate_groups do not match"): + self.create(duplicate_groups=()) + + def test_receipt_factory_rejects_nonfactory_report(self) -> None: + with self.assertRaisesRegex(TypeError, "^report must come from"): + _create_privacy_receipt(cast(PrivacyReport, object())) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_builder.py b/tests/test_release_builder.py new file mode 100644 index 0000000..69a017b --- /dev/null +++ b/tests/test_release_builder.py @@ -0,0 +1,560 @@ +from __future__ import annotations + +import gzip +import hashlib +import io +import json +import os +import shutil +import subprocess +import sys +import tarfile +import tempfile +import unittest +from pathlib import Path +from typing import Any +from unittest.mock import patch + +from tools import build_release + +EPOCH = 1_700_000_000 +SOURCE_COMMIT = "a" * 40 +ROOT = Path(__file__).resolve().parents[1] + + +def _gzip_tar( + members: tuple[tuple[str, bytes | None, int, bytes], ...], + *, + gzip_filename: str, + gzip_mtime: int, + reverse: bool = False, +) -> bytes: + tar_payload = io.BytesIO() + ordered = tuple(reversed(members)) if reverse else members + with tarfile.open( + fileobj=tar_payload, + mode="w:", + format=tarfile.PAX_FORMAT, + ) as archive: + for index, (name, payload, mode, member_type) in enumerate(ordered): + information = tarfile.TarInfo(name) + information.type = member_type + information.mode = mode + information.mtime = 111 + index + information.uid = 1000 + index + information.gid = 2000 + index + information.uname = f"user-{index}" + information.gname = f"group-{index}" + information.size = 0 if payload is None else len(payload) + information.pax_headers = {"comment": f"nondeterministic-{index}"} + archive.addfile( + information, + None if payload is None else io.BytesIO(payload), + ) + compressed = io.BytesIO() + with gzip.GzipFile( + filename=gzip_filename, + mode="wb", + compresslevel=1 if reverse else 6, + fileobj=compressed, + mtime=gzip_mtime, + ) as archive: + archive.write(tar_payload.getvalue()) + return compressed.getvalue() + + +def _safe_members() -> tuple[tuple[str, bytes | None, int, bytes], ...]: + return ( + ("netveil_audit-0.3.0/", None, 0o700, tarfile.DIRTYPE), + ( + "netveil_audit-0.3.0/README.md", + b"# Netveil\n", + 0o600, + tarfile.REGTYPE, + ), + ( + "netveil_audit-0.3.0/scripts/", + None, + 0o777, + tarfile.DIRTYPE, + ), + ( + "netveil_audit-0.3.0/scripts/netveil-audit", + b"#!/bin/sh\nexit 0\n", + 0o700, + tarfile.REGTYPE, + ), + ) + + +def _git(project: Path, *arguments: str) -> subprocess.CompletedProcess[bytes]: + environment = dict(os.environ) + environment.update( + { + "GIT_AUTHOR_EMAIL": "31526072+omar07ibrahim@users.noreply.github.com", + "GIT_AUTHOR_NAME": "Omar Ibrahim", + "GIT_COMMITTER_EMAIL": ("31526072+omar07ibrahim@users.noreply.github.com"), + "GIT_COMMITTER_NAME": "Omar Ibrahim", + } + ) + return subprocess.run( + ("git", *arguments), + cwd=project, + env=environment, + stdin=subprocess.DEVNULL, + capture_output=True, + check=True, + ) + + +def _initialize_repository(project: Path) -> str: + project.mkdir() + (project / ".gitignore").write_text("ignored.generated\n") + (project / "tracked.txt").write_text("tracked commit bytes\n") + _git(project, "init", "--quiet") + _git(project, "add", ".gitignore", "tracked.txt") + _git(project, "commit", "--quiet", "-m", "Create test fixture") + return _git(project, "rev-parse", "HEAD").stdout.decode("ascii").strip() + + +class ReleaseBuilderTests(unittest.TestCase): + def test_logically_identical_sdists_normalize_byte_identically(self) -> None: + first = _gzip_tar( + _safe_members(), + gzip_filename="first-random-name.tar", + gzip_mtime=123, + ) + second = _gzip_tar( + _safe_members(), + gzip_filename="second-random-name.tar", + gzip_mtime=987_654, + reverse=True, + ) + + normalized_first = build_release.normalize_sdist_bytes( + first, + source_date_epoch=EPOCH, + ) + normalized_second = build_release.normalize_sdist_bytes( + second, + source_date_epoch=EPOCH, + ) + self.assertEqual(normalized_first, normalized_second) + self.assertEqual(normalized_first[3], 0) + self.assertEqual( + int.from_bytes(normalized_first[4:8], "little"), + EPOCH, + ) + + with tarfile.open( + fileobj=io.BytesIO(normalized_first), + mode="r:gz", + ) as archive: + members = archive.getmembers() + self.assertEqual( + [member.name for member in members], + [ + "netveil_audit-0.3.0", + "netveil_audit-0.3.0/README.md", + "netveil_audit-0.3.0/scripts", + "netveil_audit-0.3.0/scripts/netveil-audit", + ], + ) + for member in members: + self.assertEqual(member.mtime, EPOCH) + self.assertEqual(member.uid, 0) + self.assertEqual(member.gid, 0) + self.assertEqual(member.uname, "") + self.assertEqual(member.gname, "") + self.assertEqual( + [member.mode for member in members], + [0o755, 0o644, 0o755, 0o755], + ) + + def test_unsafe_archive_members_are_rejected(self) -> None: + cases = { + "absolute": (("/absolute.txt", b"x", 0o644, tarfile.REGTYPE),), + "backslash": (("root\\escape.txt", b"x", 0o644, tarfile.REGTYPE),), + "character_device": (("root/device", None, 0o600, tarfile.CHRTYPE),), + "duplicate": ( + ("root/file.txt", b"one", 0o644, tarfile.REGTYPE), + ("root/file.txt", b"two", 0o644, tarfile.REGTYPE), + ), + "hard_link": (("root/link", None, 0o644, tarfile.LNKTYPE),), + "symlink": (("root/link", None, 0o777, tarfile.SYMTYPE),), + "traversal": (("root/../../escape.txt", b"x", 0o644, tarfile.REGTYPE),), + } + for label, members in cases.items(): + with self.subTest(label=label): + payload = _gzip_tar( + members, + gzip_filename=f"{label}.tar", + gzip_mtime=1, + ) + with self.assertRaises(build_release.ReleaseBuildError): + build_release.normalize_sdist_bytes( + payload, + source_date_epoch=EPOCH, + ) + + def test_inventory_is_stable_sorted_and_hash_bound(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + wheel = root / "netveil_audit-0.3.0-py3-none-any.whl" + sdist = root / "netveil_audit-0.3.0.tar.gz" + wheel.write_bytes(b"backend-owned-wheel-bytes") + sdist.write_bytes( + build_release.normalize_sdist_bytes( + _gzip_tar( + _safe_members(), + gzip_filename="source.tar", + gzip_mtime=99, + ), + source_date_epoch=EPOCH, + ) + ) + first = build_release.artifact_inventory( + (wheel, sdist), + source_date_epoch=EPOCH, + source_commit=SOURCE_COMMIT, + ) + second = build_release.artifact_inventory( + (sdist, wheel), + source_date_epoch=EPOCH, + source_commit=SOURCE_COMMIT, + ) + + self.assertEqual(first, second) + document: dict[str, Any] = json.loads(first) + self.assertEqual(document["schema"], build_release.INVENTORY_SCHEMA) + self.assertEqual(document["source_commit"], SOURCE_COMMIT) + self.assertEqual(document["source_date_epoch"], EPOCH) + artifacts = document["artifacts"] + self.assertEqual( + [artifact["filename"] for artifact in artifacts], + sorted((sdist.name, wheel.name)), + ) + by_name = {artifact["filename"]: artifact for artifact in artifacts} + self.assertEqual( + by_name[wheel.name]["sha256"], + hashlib.sha256(b"backend-owned-wheel-bytes").hexdigest(), + ) + self.assertEqual( + by_name[wheel.name]["size_bytes"], + len(b"backend-owned-wheel-bytes"), + ) + sdist_members = by_name[sdist.name]["members"] + self.assertEqual( + [member["path"] for member in sdist_members], + [ + "netveil_audit-0.3.0", + "netveil_audit-0.3.0/README.md", + "netveil_audit-0.3.0/scripts", + "netveil_audit-0.3.0/scripts/netveil-audit", + ], + ) + self.assertNotIn("sha256", sdist_members[0]) + self.assertEqual( + sdist_members[1]["sha256"], + hashlib.sha256(b"# Netveil\n").hexdigest(), + ) + + def test_normalization_never_overwrites_an_existing_destination(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "input" / "netveil_audit-0.3.0.tar.gz" + destination = root / "output" / source.name + source.parent.mkdir() + destination.parent.mkdir() + source.write_bytes( + _gzip_tar( + _safe_members(), + gzip_filename="source.tar", + gzip_mtime=99, + ) + ) + destination.write_bytes(b"retained-existing-artifact") + + with self.assertRaises(build_release.ReleaseBuildError): + build_release.normalize_sdist( + source, + destination, + source_date_epoch=EPOCH, + ) + self.assertEqual( + destination.read_bytes(), + b"retained-existing-artifact", + ) + + def test_sanitized_environment_is_complete_and_reproducible(self) -> None: + home = Path("/private/netveil-build-home") + first = build_release.sanitized_build_environment( + source_date_epoch=EPOCH, + home=home, + ) + second = build_release.sanitized_build_environment( + source_date_epoch=EPOCH, + home=home, + ) + self.assertEqual(first, second) + self.assertEqual(first["SOURCE_DATE_EPOCH"], str(EPOCH)) + self.assertEqual(first["HOME"], str(home)) + self.assertEqual(first["PYTHONHASHSEED"], "0") + for excluded in ( + "GIT_CONFIG_GLOBAL", + "LD_PRELOAD", + "PYTHONHOME", + "PYTHONPATH", + ): + self.assertNotIn(excluded, first) + + def test_pinned_backend_accepts_the_live_project_metadata(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "project" + project.mkdir() + for name in ("LICENSE", "README.md", "pyproject.toml"): + shutil.copy2(ROOT / name, project / name) + for name in ("scripts", "src"): + shutil.copytree(ROOT / name, project / name) + home = Path(temporary) / "home" + home.mkdir() + + completed = subprocess.run( + ( + sys.executable, + "-c", + ( + "from setuptools import build_meta;" + "build_meta.get_requires_for_build_sdist()" + ), + ), + cwd=project, + env=dict( + build_release.sanitized_build_environment( + source_date_epoch=EPOCH, + home=home, + ) + ), + stdin=subprocess.DEVNULL, + capture_output=True, + check=False, + timeout=30, + ) + + self.assertEqual( + completed.returncode, + 0, + msg=(completed.stdout + completed.stderr).decode( + "utf-8", + errors="replace", + ), + ) + + def test_git_source_commit_rejects_tracked_and_untracked_changes(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + project = root / "project" + expected_commit = _initialize_repository(project) + home = root / "home" + home.mkdir() + + self.assertEqual( + build_release._git_source_commit(project.resolve(), home), + expected_commit, + ) + tracked = project / "tracked.txt" + tracked.write_text("dirty tracked bytes\n") + with self.assertRaisesRegex( + build_release.ReleaseBuildError, + "^netveil_release_error:worktree_dirty$", + ): + build_release._git_source_commit(project.resolve(), home) + tracked.write_text("tracked commit bytes\n") + + untracked = project / "untracked.txt" + untracked.write_text("dirty untracked bytes\n") + with self.assertRaisesRegex( + build_release.ReleaseBuildError, + "^netveil_release_error:worktree_dirty$", + ): + build_release._git_source_commit(project.resolve(), home) + + def test_exported_source_is_exact_head_and_excludes_ignored_state(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + project = root / "project" + expected_commit = _initialize_repository(project) + (project / "ignored.generated").write_text( + "must not influence the backend\n" + ) + home = root / "home" + home.mkdir() + snapshot = root / "snapshot" + + observed_commit = build_release._export_clean_head( + project.resolve(), + snapshot, + root / "source.tar", + home, + ) + + self.assertEqual(observed_commit, expected_commit) + self.assertEqual( + (snapshot / "tracked.txt").read_text(), + "tracked commit bytes\n", + ) + self.assertEqual( + (snapshot / ".gitignore").read_text(), + "ignored.generated\n", + ) + self.assertFalse((snapshot / "ignored.generated").exists()) + self.assertFalse((snapshot / ".git").exists()) + + def test_release_destination_must_be_outside_source_tree(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "project" + project.mkdir() + + with self.assertRaisesRegex( + build_release.ReleaseBuildError, + "^netveil_release_error:build_path_invalid$", + ): + build_release.build_release( + project, + project / "release", + source_date_epoch=EPOCH, + python_executable=Path(sys.executable), + ) + + def test_build_python_preserves_a_virtual_environment_symlink(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + binary = root / "venv/bin" + binary.mkdir(parents=True) + launcher = binary / "python" + launcher.symlink_to(Path(sys.executable)) + + observed = build_release._build_python_path(launcher) + dereferenced = launcher.resolve() + + self.assertEqual(observed, launcher.absolute()) + self.assertNotEqual(observed, dereferenced) + + def test_backend_orchestration_is_mocked_and_wheel_bytes_are_untouched( + self, + ) -> None: + backend_wheel = b"opaque-backend-wheel-bytes" + backend_sdist = _gzip_tar( + _safe_members(), + gzip_filename="backend-random-name.tar", + gzip_mtime=42, + ) + observed_commands: list[tuple[str, ...]] = [] + observed_environments: list[dict[str, str]] = [] + observed_working_directories: list[Path] = [] + + def fake_export( + project: Path, + destination: Path, + archive_path: Path, + home: Path, + ) -> str: + del project, archive_path, home + destination.mkdir() + return SOURCE_COMMIT + + def fake_run( + command: tuple[str, ...], + *, + cwd: Path, + env: dict[str, str], + stdin: int, + capture_output: bool, + check: bool, + timeout: int, + ) -> subprocess.CompletedProcess[bytes]: + del stdin, capture_output, check, timeout + observed_commands.append(command) + observed_environments.append(env) + observed_working_directories.append(cwd) + backend = Path(command[command.index("--outdir") + 1]) + (backend / "netveil_audit-0.3.0-py3-none-any.whl").write_bytes( + backend_wheel + ) + (backend / "netveil_audit-0.3.0.tar.gz").write_bytes(backend_sdist) + return subprocess.CompletedProcess( + command, + 0, + stdout=b"", + stderr=b"", + ) + + with tempfile.TemporaryDirectory() as temporary: + project = Path(temporary) / "project" + project.mkdir() + output = Path(temporary) / "release" + with ( + patch( + "tools.build_release._export_clean_head", + side_effect=fake_export, + ), + patch( + "tools.build_release._git_source_commit", + return_value=SOURCE_COMMIT, + ), + patch( + "tools.build_release.subprocess.run", + side_effect=fake_run, + ), + ): + inventory = build_release.build_release( + project, + output, + source_date_epoch=EPOCH, + python_executable=Path(sys.executable), + ) + + wheel = output / "netveil_audit-0.3.0-py3-none-any.whl" + normalized_sdist = output / "netveil_audit-0.3.0.tar.gz" + self.assertEqual(wheel.read_bytes(), backend_wheel) + self.assertEqual( + normalized_sdist.read_bytes(), + build_release.normalize_sdist_bytes( + backend_sdist, + source_date_epoch=EPOCH, + ), + ) + self.assertEqual( + (output / build_release.INVENTORY_FILENAME).read_bytes(), + inventory, + ) + self.assertEqual(json.loads(inventory)["source_commit"], SOURCE_COMMIT) + + self.assertEqual(len(observed_commands), 1) + command = observed_commands[0] + self.assertEqual( + command[:4], + ( + os.path.abspath(sys.executable), + "-m", + "build", + "--no-isolation", + ), + ) + backend_path = Path(command[command.index("--outdir") + 1]) + self.assertFalse(backend_path.is_relative_to(project)) + self.assertEqual(observed_working_directories, [Path(command[-1])]) + self.assertFalse(observed_working_directories[0].is_relative_to(project)) + self.assertEqual( + observed_environments[0], + dict( + build_release.sanitized_build_environment( + source_date_epoch=EPOCH, + home=Path(observed_environments[0]["HOME"]), + ) + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_visuals.py b/tests/test_visuals.py new file mode 100644 index 0000000..f781311 --- /dev/null +++ b/tests/test_visuals.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import json +import unittest +from pathlib import Path +from xml.etree import ElementTree + +from tools import render_architecture, render_evidence + +ROOT = Path(__file__).resolve().parents[1] + + +class VisualEvidenceTests(unittest.TestCase): + def test_architecture_svg_is_current_and_code_derived(self) -> None: + expected = render_architecture.render() + observed = (ROOT / "docs" / "assets" / "architecture.svg").read_bytes() + self.assertEqual(observed, expected) + text = observed.decode("utf-8") + self.assertIn("Netveil installed execution boundary", text) + self.assertIn("-I -E -S -B", text) + self.assertIn("installed RECORD is not a signature", text) + + def test_committed_release_evidence_views_are_current(self) -> None: + evidence_paths = ( + render_evidence.VERIFICATION_PATH, + render_evidence.INVENTORY_PATH, + ) + if not all(path.is_file() for path in evidence_paths): + self.assertFalse( + any(path.exists() for path in evidence_paths), + "release evidence must be either complete or absent", + ) + return + + outputs = render_evidence.render_bundle( + render_evidence.VERIFICATION_PATH.read_bytes(), + render_evidence.INVENTORY_PATH.read_bytes(), + generator_payload=render_evidence.GENERATOR_PATH.read_bytes(), + ) + for relative_path, expected in outputs.items(): + observed = (ROOT / relative_path).read_bytes() + self.assertEqual(observed, expected, relative_path) + + for relative_path in ( + render_evidence.CLI_SVG_PATH, + render_evidence.COUNTS_SVG_PATH, + render_evidence.MATRIX_SVG_PATH, + render_evidence.PROVENANCE_SVG_PATH, + ): + ElementTree.fromstring(outputs[relative_path]) + for line in outputs[render_evidence.CAST_PATH].splitlines(): + json.loads(line) + + def test_readme_presents_every_reproducible_visual(self) -> None: + readme = (ROOT / "README.md").read_text() + expected_links = { + "docs/assets/architecture.svg", + *render_evidence.VISUAL_OUTPUT_PATHS, + render_evidence.MANIFEST_PATH, + "docs/evidence/fresh-wheel-verification.json", + "docs/evidence/release-inventory.json", + } + for link in expected_links: + self.assertIn(link, readme) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..dd9ff35 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ +"""Repository-local verification tools for Netveil.""" diff --git a/tools/build_release.py b/tools/build_release.py new file mode 100755 index 0000000..0ff0aff --- /dev/null +++ b/tools/build_release.py @@ -0,0 +1,821 @@ +#!/usr/bin/env python3 +"""Build and normalize a deterministic Netveil release with the standard library. + +The build backend owns wheel bytes: this helper copies the wheel byte-for-byte +and never opens or repacks it. The setuptools sdist is treated as an untrusted +tar.gz container and rewritten into one canonical archive before publication. +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import io +import json +import os +import shutil +import stat +import subprocess +import sys +import tarfile +import tempfile +import zlib +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Final, NoReturn + +INVENTORY_SCHEMA: Final = "netveil.release-inventory.v1" +INVENTORY_FILENAME: Final = "release-inventory.json" + +_MAX_COMPRESSED_SDIST_BYTES: Final = 128 * 1_048_576 +_MAX_TAR_BYTES: Final = 256 * 1_048_576 +_MAX_MEMBER_BYTES: Final = 64 * 1_048_576 +_MAX_TOTAL_FILE_BYTES: Final = 192 * 1_048_576 +_MAX_MEMBERS: Final = 20_000 +_MAX_ARTIFACT_BYTES: Final = 512 * 1_048_576 +_BUILD_TIMEOUT_SECONDS: Final = 10 * 60 +_GIT_TIMEOUT_SECONDS: Final = 60 +_MAX_GIT_OUTPUT_BYTES: Final = 1_048_576 +_TAR_BLOCK_BYTES: Final = 512 +_TAR_TRAILER_BYTES: Final = 2 * _TAR_BLOCK_BYTES +_GZIP_MAX_MTIME: Final = (1 << 32) - 1 +_LOWER_HEX: Final = frozenset("0123456789abcdef") + + +class ReleaseBuildError(RuntimeError): + """A stable release failure that does not disclose archive contents.""" + + __slots__ = ("code",) + + code: str + + def __init__(self, code: str) -> None: + self.code = code + super().__init__(f"netveil_release_error:{code}") + + +@dataclass(frozen=True, slots=True) +class _CanonicalMember: + """One validated logical member ready for canonical tar serialization.""" + + name: str + payload: bytes | None + executable: bool + + @property + def is_directory(self) -> bool: + return self.payload is None + + +def _fail(code: str) -> NoReturn: + raise ReleaseBuildError(code) + + +def _validated_epoch(source_date_epoch: int) -> int: + if ( + type(source_date_epoch) is not int + or not 0 <= source_date_epoch <= _GZIP_MAX_MTIME + ): + _fail("source_date_epoch_invalid") + return source_date_epoch + + +def _validated_source_commit(source_commit: str) -> str: + if ( + type(source_commit) is not str + or len(source_commit) != 40 + or any(character not in _LOWER_HEX for character in source_commit) + ): + _fail("source_commit_invalid") + return source_commit + + +def _canonical_member_name(member: tarfile.TarInfo) -> str: + raw_name = member.name + if ( + type(raw_name) is not str + or not raw_name + or "\\" in raw_name + or "\x00" in raw_name + or any(ord(character) < 32 or ord(character) == 127 for character in raw_name) + ): + _fail("sdist_member_unsafe") + normalized = ( + raw_name[:-1] if member.isdir() and raw_name.endswith("/") else raw_name + ) + path = PurePosixPath(normalized) + first = path.parts[0] if path.parts else "" + if ( + not normalized + or path.is_absolute() + or path.as_posix() != normalized + or any(part in ("", ".", "..") for part in path.parts) + or (len(first) >= 2 and first[0].isalpha() and first[1] == ":") + or (not member.isdir() and raw_name.endswith("/")) + ): + _fail("sdist_member_unsafe") + return normalized + + +def _decompress_sdist(payload: bytes) -> bytes: + if ( + type(payload) is not bytes + or not 0 < len(payload) <= _MAX_COMPRESSED_SDIST_BYTES + ): + _fail("sdist_invalid") + source = io.BytesIO(payload) + try: + with gzip.GzipFile(fileobj=source, mode="rb") as compressed: + tar_payload = compressed.read(_MAX_TAR_BYTES + 1) + except (EOFError, OSError, gzip.BadGzipFile, zlib.error): + _fail("sdist_invalid") + if ( + not 0 < len(tar_payload) <= _MAX_TAR_BYTES + or len(tar_payload) % _TAR_BLOCK_BYTES + or len(tar_payload) < _TAR_TRAILER_BYTES + ): + _fail("sdist_invalid") + return tar_payload + + +def _read_members(tar_payload: bytes) -> tuple[_CanonicalMember, ...]: + observed: dict[str, _CanonicalMember] = {} + total_file_bytes = 0 + archive_offset = -1 + try: + with tarfile.open(fileobj=io.BytesIO(tar_payload), mode="r:") as archive: + members = archive.getmembers() + archive_offset = archive.offset + if not 1 <= len(members) <= _MAX_MEMBERS: + _fail("sdist_invalid") + for member in members: + if not (member.isdir() or member.isreg()): + _fail("sdist_member_unsafe") + if member.sparse is not None: + _fail("sdist_member_unsafe") + name = _canonical_member_name(member) + if name in observed: + _fail("sdist_member_duplicate") + if member.isdir(): + if member.size != 0: + _fail("sdist_member_unsafe") + observed[name] = _CanonicalMember( + name=name, + payload=None, + executable=True, + ) + continue + if not 0 <= member.size <= _MAX_MEMBER_BYTES: + _fail("sdist_member_unsafe") + total_file_bytes += member.size + if total_file_bytes > _MAX_TOTAL_FILE_BYTES: + _fail("sdist_invalid") + extracted = archive.extractfile(member) + if extracted is None: + _fail("sdist_invalid") + content = extracted.read(_MAX_MEMBER_BYTES + 1) + if len(content) != member.size: + _fail("sdist_invalid") + observed[name] = _CanonicalMember( + name=name, + payload=content, + executable=bool(member.mode & 0o111), + ) + except ReleaseBuildError: + raise + except (OSError, tarfile.TarError, UnicodeError, ValueError): + _fail("sdist_invalid") + + if ( + archive_offset < 0 + or len(tar_payload) - archive_offset < _TAR_TRAILER_BYTES + or any(tar_payload[archive_offset:]) + ): + _fail("sdist_invalid") + return tuple(observed[name] for name in sorted(observed)) + + +def _serialize_tar( + members: tuple[_CanonicalMember, ...], + *, + source_date_epoch: int, +) -> bytes: + destination = io.BytesIO() + try: + with tarfile.open( + fileobj=destination, + mode="w:", + format=tarfile.USTAR_FORMAT, + ) as archive: + for member in members: + information = tarfile.TarInfo( + member.name + "/" if member.is_directory else member.name + ) + information.type = ( + tarfile.DIRTYPE if member.is_directory else tarfile.REGTYPE + ) + information.size = 0 if member.payload is None else len(member.payload) + information.mode = ( + 0o755 if member.is_directory or member.executable else 0o644 + ) + information.mtime = source_date_epoch + information.uid = 0 + information.gid = 0 + information.uname = "" + information.gname = "" + information.linkname = "" + information.pax_headers = {} + archive.addfile( + information, + None if member.payload is None else io.BytesIO(member.payload), + ) + except (OSError, tarfile.TarError, UnicodeError, ValueError): + _fail("sdist_not_canonicalizable") + return destination.getvalue() + + +def normalize_sdist_bytes( + payload: bytes, + *, + source_date_epoch: int, +) -> bytes: + """Return one canonical gzip-compressed USTAR sdist.""" + + epoch = _validated_epoch(source_date_epoch) + members = _read_members(_decompress_sdist(payload)) + canonical_tar = _serialize_tar(members, source_date_epoch=epoch) + destination = io.BytesIO() + try: + with gzip.GzipFile( + filename="", + mode="wb", + compresslevel=9, + fileobj=destination, + mtime=epoch, + ) as compressed: + compressed.write(canonical_tar) + except (OSError, ValueError, zlib.error): + _fail("sdist_not_canonicalizable") + return destination.getvalue() + + +def _read_regular_file(path: Path, *, maximum: int) -> bytes: + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_nlink != 1 + or not 0 < before.st_size <= maximum + ): + _fail("artifact_invalid") + chunks: list[bytes] = [] + consumed = 0 + while consumed < before.st_size: + chunk = os.read(descriptor, min(1_048_576, before.st_size - consumed)) + if not chunk: + _fail("artifact_invalid") + chunks.append(chunk) + consumed += len(chunk) + after = os.fstat(descriptor) + if consumed != before.st_size or ( + after.st_dev, + after.st_ino, + after.st_mode, + after.st_size, + after.st_mtime_ns, + after.st_ctime_ns, + ) != ( + before.st_dev, + before.st_ino, + before.st_mode, + before.st_size, + before.st_mtime_ns, + before.st_ctime_ns, + ): + _fail("artifact_invalid") + return b"".join(chunks) + except ReleaseBuildError: + raise + except OSError: + _fail("artifact_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + +def _write_new_file(path: Path, payload: bytes, *, mode: int = 0o644) -> None: + descriptor = -1 + created = False + try: + descriptor = os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC | os.O_NOFOLLOW, + mode, + ) + created = True + offset = 0 + while offset < len(payload): + written = os.write(descriptor, payload[offset:]) + if written <= 0: + raise OSError + offset += written + os.fchmod(descriptor, mode) + os.fsync(descriptor) + if stat.S_IMODE(os.fstat(descriptor).st_mode) != mode or os.fstat( + descriptor + ).st_size != len(payload): + raise OSError + except OSError: + if created: + try: + path.unlink(missing_ok=True) + except OSError: + pass + _fail("artifact_write_failed") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + +def normalize_sdist( + source: Path, + destination: Path, + *, + source_date_epoch: int, +) -> None: + """Normalize one sdist path into a new destination file.""" + + if not source.name.endswith(".tar.gz") or destination.name != source.name: + _fail("sdist_path_invalid") + payload = _read_regular_file(source, maximum=_MAX_COMPRESSED_SDIST_BYTES) + normalized = normalize_sdist_bytes( + payload, + source_date_epoch=source_date_epoch, + ) + _write_new_file(destination, normalized) + + +def _sdist_member_inventory(payload: bytes) -> list[dict[str, object]]: + members = _read_members(_decompress_sdist(payload)) + inventory: list[dict[str, object]] = [] + for member in members: + if member.is_directory: + inventory.append( + { + "kind": "directory", + "mode": "0755", + "path": member.name, + "size_bytes": 0, + } + ) + else: + assert member.payload is not None + inventory.append( + { + "kind": "file", + "mode": "0755" if member.executable else "0644", + "path": member.name, + "sha256": hashlib.sha256(member.payload).hexdigest(), + "size_bytes": len(member.payload), + } + ) + return inventory + + +def artifact_inventory( + artifacts: Sequence[Path], + *, + source_date_epoch: int, + source_commit: str, +) -> bytes: + """Render canonical unsigned integrity evidence for release artifacts. + + The inventory binds artifact bytes to a builder-observed Git commit. It is + deliberately not represented as a signature or third-party attestation. + """ + + epoch = _validated_epoch(source_date_epoch) + commit = _validated_source_commit(source_commit) + records: dict[str, dict[str, object]] = {} + for path in artifacts: + if not isinstance(path, Path): + _fail("artifact_invalid") + name = path.name + if not name or name in records or PurePosixPath(name).name != name: + _fail("artifact_invalid") + if name.endswith(".whl"): + kind = "wheel" + elif name.endswith(".tar.gz"): + kind = "sdist" + else: + _fail("artifact_invalid") + payload = _read_regular_file(path, maximum=_MAX_ARTIFACT_BYTES) + record: dict[str, object] = { + "filename": name, + "kind": kind, + "sha256": hashlib.sha256(payload).hexdigest(), + "size_bytes": len(payload), + } + if kind == "sdist": + record["members"] = _sdist_member_inventory(payload) + records[name] = record + if len(records) != 2 or {record["kind"] for record in records.values()} != { + "wheel", + "sdist", + }: + _fail("artifact_invalid") + document = { + "artifacts": [records[name] for name in sorted(records)], + "schema": INVENTORY_SCHEMA, + "source_commit": commit, + "source_date_epoch": epoch, + } + return ( + json.dumps( + document, + ensure_ascii=True, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("ascii") + + b"\n" + ) + + +def sanitized_build_environment( + *, + source_date_epoch: int, + home: Path, +) -> Mapping[str, str]: + """Return the complete, minimal environment used for the backend process.""" + + epoch = _validated_epoch(source_date_epoch) + if not home.is_absolute(): + _fail("build_path_invalid") + return { + "HOME": str(home), + "LANG": "C", + "LC_ALL": "C", + "PATH": os.defpath, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INDEX": "1", + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONHASHSEED": "0", + "PYTHONNOUSERSITE": "1", + "SOURCE_DATE_EPOCH": str(epoch), + "TZ": "UTC", + } + + +def _git_binary() -> Path: + located = shutil.which("git", path=os.defpath) + if located is None: + _fail("git_unavailable") + try: + path = Path(located).resolve(strict=True) + status = path.stat() + except (OSError, RuntimeError): + _fail("git_unavailable") + if ( + not stat.S_ISREG(status.st_mode) + or not status.st_mode & stat.S_IXUSR + or status.st_mode & (stat.S_ISUID | stat.S_ISGID) + ): + _fail("git_unavailable") + return path + + +def _git_environment(home: Path) -> Mapping[str, str]: + if not home.is_absolute(): + _fail("build_path_invalid") + return { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_OPTIONAL_LOCKS": "0", + "GIT_TERMINAL_PROMPT": "0", + "HOME": str(home), + "LANG": "C", + "LC_ALL": "C", + "PATH": os.defpath, + } + + +def _run_git( + project: Path, + home: Path, + arguments: Sequence[str], +) -> bytes: + try: + completed = subprocess.run( + (str(_git_binary()), *arguments), + cwd=project, + env=dict(_git_environment(home)), + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + check=False, + timeout=_GIT_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired): + _fail("source_repository_invalid") + if completed.returncode != 0 or len(completed.stdout) > _MAX_GIT_OUTPUT_BYTES: + _fail("source_repository_invalid") + return completed.stdout + + +def _git_source_commit(project: Path, home: Path) -> str: + top_level_payload = _run_git( + project, + home, + ("rev-parse", "--show-toplevel"), + ) + try: + top_level_text = top_level_payload.decode(sys.getfilesystemencoding()) + top_level = Path(top_level_text.removesuffix("\n")).resolve(strict=True) + except (OSError, RuntimeError, UnicodeError): + _fail("source_repository_invalid") + if ( + not top_level_payload.endswith(b"\n") + or top_level_payload.endswith(b"\n\n") + or top_level != project + ): + _fail("source_repository_invalid") + + commit_payload = _run_git( + project, + home, + ("rev-parse", "--verify", "HEAD^{commit}"), + ) + try: + commit = commit_payload.decode("ascii").removesuffix("\n") + except UnicodeError: + _fail("source_repository_invalid") + if ( + not commit_payload.endswith(b"\n") + or commit_payload.endswith(b"\n\n") + or len(commit_payload) != 41 + ): + _fail("source_repository_invalid") + try: + validated_commit = _validated_source_commit(commit) + except ReleaseBuildError: + _fail("source_repository_invalid") + + status = _run_git( + project, + home, + ("status", "--porcelain=v1", "--untracked-files=all"), + ) + if status: + _fail("worktree_dirty") + return validated_commit + + +def _materialize_source_snapshot( + archive_payload: bytes, + destination: Path, +) -> None: + members = _read_members(archive_payload) + if not members: + _fail("source_archive_invalid") + try: + destination.mkdir(mode=0o700) + for member in members: + path = destination / member.name + if member.is_directory: + path.mkdir(mode=0o755, parents=True, exist_ok=False) + else: + path.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + assert member.payload is not None + _write_new_file( + path, + member.payload, + mode=0o755 if member.executable else 0o644, + ) + except ReleaseBuildError: + raise + except OSError: + _fail("source_archive_invalid") + + +def _export_clean_head( + project: Path, + destination: Path, + archive_path: Path, + home: Path, +) -> str: + commit = _git_source_commit(project, home) + if ( + destination.exists() + or archive_path.exists() + or destination.is_relative_to(project) + or archive_path.is_relative_to(project) + ): + _fail("build_path_invalid") + _run_git( + project, + home, + ( + "archive", + "--format=tar", + f"--output={archive_path}", + commit, + ), + ) + archive_payload = _read_regular_file( + archive_path, + maximum=_MAX_TAR_BYTES, + ) + _materialize_source_snapshot(archive_payload, destination) + try: + archive_path.unlink() + except OSError: + _fail("source_archive_invalid") + return commit + + +def _backend_artifacts(directory: Path) -> tuple[Path, Path]: + try: + entries = tuple(sorted(directory.iterdir(), key=lambda path: path.name)) + except OSError: + _fail("backend_output_invalid") + if len(entries) != 2 or any( + entry.is_symlink() or not entry.is_file() for entry in entries + ): + _fail("backend_output_invalid") + wheels = tuple(path for path in entries if path.name.endswith(".whl")) + sdists = tuple(path for path in entries if path.name.endswith(".tar.gz")) + if len(wheels) != 1 or len(sdists) != 1: + _fail("backend_output_invalid") + return wheels[0], sdists[0] + + +def _build_python_path(python_executable: Path | None) -> Path: + """Keep a virtual-environment launcher path without dereferencing it.""" + + candidate = Path(sys.executable) if python_executable is None else python_executable + try: + python = Path(os.path.abspath(candidate)) + except (OSError, TypeError, ValueError): + _fail("build_path_invalid") + if not python.is_absolute() or not python.is_file(): + _fail("build_path_invalid") + return python + + +def build_release( + project_root: Path, + output_directory: Path, + *, + source_date_epoch: int, + python_executable: Path | None = None, +) -> bytes: + """Run the backend once and atomically publish wheel, sdist, and inventory.""" + + epoch = _validated_epoch(source_date_epoch) + try: + project = project_root.resolve(strict=True) + output = output_directory.resolve(strict=False) + except (OSError, RuntimeError): + _fail("build_path_invalid") + if ( + not project.is_dir() + or not output.is_absolute() + or output.exists() + or output.is_symlink() + or output == project + or output.is_relative_to(project) + ): + _fail("build_path_invalid") + python = _build_python_path(python_executable) + try: + with tempfile.TemporaryDirectory( + prefix="netveil-build-", + ) as temporary: + workspace = Path(temporary) + backend = workspace / "backend" + home = workspace / "home" + source = workspace / "source" + archive_path = workspace / "source.tar" + for directory in (backend, home): + directory.mkdir(mode=0o700) + source_commit = _export_clean_head( + project, + source, + archive_path, + home, + ) + command = ( + str(python), + "-m", + "build", + "--no-isolation", + "--outdir", + str(backend), + str(source), + ) + try: + completed = subprocess.run( + command, + cwd=source, + env=dict( + sanitized_build_environment( + source_date_epoch=epoch, + home=home, + ) + ), + stdin=subprocess.DEVNULL, + capture_output=True, + check=False, + timeout=_BUILD_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired): + _fail("backend_failed") + if completed.returncode != 0: + _fail("backend_failed") + if _git_source_commit(project, home) != source_commit: + _fail("source_commit_changed") + wheel, sdist = _backend_artifacts(backend) + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists() or output.is_symlink(): + _fail("build_path_invalid") + with tempfile.TemporaryDirectory( + prefix=f".{output.name}.publish-", + dir=output.parent, + ) as publication_temporary: + release = Path(publication_temporary) / "release" + release.mkdir(mode=0o700) + wheel_payload = _read_regular_file( + wheel, + maximum=_MAX_ARTIFACT_BYTES, + ) + _write_new_file(release / wheel.name, wheel_payload) + normalize_sdist( + sdist, + release / sdist.name, + source_date_epoch=epoch, + ) + inventory = artifact_inventory( + (release / wheel.name, release / sdist.name), + source_date_epoch=epoch, + source_commit=source_commit, + ) + _write_new_file(release / INVENTORY_FILENAME, inventory) + os.rename(release, output) + return inventory + except ReleaseBuildError: + raise + except OSError: + _fail("artifact_write_failed") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Build a wheel and canonical sdist without dependency isolation, " + "then emit deterministic SHA-256 inventory evidence." + ) + ) + parser.add_argument("project_root", type=Path) + parser.add_argument("output_directory", type=Path) + parser.add_argument( + "--source-date-epoch", + type=int, + required=True, + ) + parser.add_argument("--python", type=Path) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + namespace = _parser().parse_args(argv) + try: + inventory = build_release( + namespace.project_root, + namespace.output_directory, + source_date_epoch=namespace.source_date_epoch, + python_executable=namespace.python, + ) + except ReleaseBuildError as error: + print(str(error), file=sys.stderr) + return 2 + sys.stdout.buffer.write(inventory) + sys.stdout.buffer.flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/render_architecture.py b/tools/render_architecture.py new file mode 100755 index 0000000..0ac6509 --- /dev/null +++ b/tools/render_architecture.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""Render the code-derived Netveil execution-boundary diagram.""" + +from __future__ import annotations + +import argparse +import ast +import html +from pathlib import Path +from typing import Final + +ROOT: Final = Path(__file__).resolve().parents[1] +OUTPUT: Final = ROOT / "docs" / "assets" / "architecture.svg" + + +def _integer(expression: ast.expr) -> int: + if isinstance(expression, ast.Constant) and type(expression.value) is int: + return expression.value + if isinstance(expression, ast.BinOp) and isinstance(expression.op, ast.Mult): + return _integer(expression.left) * _integer(expression.right) + raise ValueError("unsupported integer constant") + + +def _assignments(path: Path) -> dict[str, ast.expr]: + tree = ast.parse(path.read_bytes(), filename=str(path)) + values: dict[str, ast.expr] = {} + for node in tree.body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name): + values[target.id] = node.value + elif isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name): + if node.value is not None: + values[node.target.id] = node.value + return values + + +def _source_contract() -> tuple[str, int, int, int, tuple[str, ...]]: + launcher = (ROOT / "scripts" / "netveil-audit").read_text() + handoff = 'exec "$netveil_script_directory/python" -IESB "$0" "$@"' + if handoff not in launcher: + raise ValueError("launcher isolation handoff changed") + + parser = _assignments(ROOT / "src" / "netveil" / "parser.py") + cli = _assignments(ROOT / "src" / "netveil" / "cli.py") + bootstrap = _assignments(ROOT / "src" / "netveil_bootstrap.py") + source_modules = ast.literal_eval(bootstrap["_SOURCE_MODULES"]) + module_names = tuple(module for module, _, _ in source_modules) + return ( + "-I -E -S -B", + _integer(parser["MAX_INPUT_BYTES"]), + _integer(parser["MAX_PHYSICAL_LINES"]), + _integer(cli["_MAX_KEY_BYTES"]), + module_names, + ) + + +def _text( + x: int, + y: int, + value: str, + *, + size: int = 16, + weight: int = 400, + fill: str = "#d8e3f0", + anchor: str = "start", +) -> str: + return ( + f'{html.escape(value)}' + ) + + +def _box( + x: int, + y: int, + width: int, + height: int, + *, + fill: str, + stroke: str, + title: str, + lines: tuple[str, ...], +) -> str: + content = [ + ( + f'' + ), + _text(x + 22, y + 34, title, size=18, weight=700, fill="#f8fbff"), + ] + for index, line in enumerate(lines): + content.append( + _text( + x + 22, + y + 64 + index * 24, + line, + size=14, + fill="#c7d3e0", + ) + ) + return "\n".join(content) + + +def render() -> bytes: + flags, max_bytes, max_lines, max_key, modules = _source_contract() + module_label = ", ".join(modules) + mib = max_bytes // (1024 * 1024) + svg = [ + ( + '' + ), + 'Netveil installed execution and trust boundary', + ( + 'Code-derived architecture from the installed ' + "polyglot launcher through pinned source loading to a pseudonymized " + "receipt." + ), + "", + ( + '' + '' + '' + "" + ), + ( + '' + '' + "" + ), + "", + '', + _text(70, 66, "Netveil installed execution boundary", size=34, weight=750), + _text( + 70, + 99, + "Generated from the 0.3.0 launcher, parser, CLI and bootstrap sources", + size=16, + fill="#8fa7bd", + ), + _box( + 70, + 145, + 390, + 170, + fill="#33251a", + stroke="#f59e0b", + title="1 Trusted startup", + lines=( + "/bin/sh → sibling CPython", + f"combined flags: {flags}", + "no site hooks • env ignored • safe path", + "launcher + installed RECORD are trust roots", + ), + ), + _box( + 525, + 145, + 390, + 170, + fill="#142a32", + stroke="#22d3ee", + title="2 Artifact drift guard", + lines=( + "launcher and bootstrap: installed hash + size", + "bounded O_NOFOLLOW descriptor reads", + "identity checked before and after read", + "installed RECORD is not a signature", + ), + ), + _box( + 980, + 145, + 390, + 170, + fill="#17273a", + stroke="#60a5fa", + title="3 Closed source loader", + lines=( + f"{len(modules)} allowed modules", + "pinned source bytes compiled in memory", + "unknown netveil.* imports rejected", + "package bytecode never executes", + ), + ), + ( + '' + ), + ( + '' + ), + _box( + 70, + 395, + 600, + 205, + fill="#1f2038", + stroke="#a78bfa", + title="4 Sensitive local inputs", + lines=( + f"corpus: regular file • ≤ {mib} MiB • ≤ {max_lines:,} lines", + f"key: owner-only regular file • 32–{max_key:,} exact bytes", + "same inode or byte-identical corpus/key rejected", + "sensitive values omitted from handled output", + "no DNS, socket or child-process workflow", + ), + ), + _box( + 770, + 395, + 600, + 205, + fill="#173126", + stroke="#34d399", + title="5 Public deterministic result", + lines=( + "parse → canonicalize → aggregate", + "domain-separated HMAC-SHA256 identifiers", + "fixed-schema sorted-key canonical JSON", + "receipt digest binds the public report", + "pseudonymized, not anonymous", + ), + ), + ( + '' + ), + ( + '' + ), + ( + '' + ), + _text(94, 704, "Exact source inventory", size=18, weight=700, fill="#f8fbff"), + _text(94, 739, module_label, size=14, fill="#9fb7cc"), + _text(94, 792, "Trust-root legend", size=16, weight=700, fill="#fbbf24"), + _text( + 94, + 823, + "amber = executes before or supplies the mutable consistency boundary", + size=14, + fill="#c7d3e0", + ), + _text(760, 792, "Verified-byte legend", size=16, weight=700, fill="#67e8f9"), + _text( + 760, + 823, + "cyan / blue = checked before compilation by the supported installed path", + size=14, + fill="#c7d3e0", + ), + _text( + 70, + 935, + "Reproduce: python3 tools/render_architecture.py --check", + size=14, + fill="#6f8aa3", + ), + _text( + 1370, + 935, + "No network data or endpoint fixture is used by this diagram", + size=14, + fill="#6f8aa3", + anchor="end", + ), + "", + ] + return ("\n".join(svg) + "\n").encode() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + action="store_true", + help="fail when the committed SVG differs from generated bytes", + ) + arguments = parser.parse_args() + payload = render() + if arguments.check: + if not OUTPUT.is_file() or OUTPUT.read_bytes() != payload: + raise SystemExit("architecture.svg is stale") + return 0 + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + OUTPUT.write_bytes(payload) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/render_evidence.py b/tools/render_evidence.py new file mode 100755 index 0000000..ff9dded --- /dev/null +++ b/tools/render_evidence.py @@ -0,0 +1,1220 @@ +#!/usr/bin/env python3 +"""Render Netveil's committed release evidence as deterministic inert visuals.""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import os +import stat +import sys +import tempfile +import textwrap +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, NoReturn, cast + +ROOT: Final = Path(__file__).resolve().parents[1] +VERIFICATION_PATH: Final = ROOT / "docs/evidence/fresh-wheel-verification.json" +INVENTORY_PATH: Final = ROOT / "docs/evidence/release-inventory.json" +GENERATOR_PATH: Final = ROOT / "tools/render_evidence.py" + +CLI_SVG_PATH: Final = "docs/assets/cli-session.svg" +COUNTS_SVG_PATH: Final = "docs/assets/receipt-counts.svg" +MATRIX_SVG_PATH: Final = "docs/assets/verification-matrix.svg" +PROVENANCE_SVG_PATH: Final = "docs/assets/artifact-provenance.svg" +CAST_PATH: Final = "docs/evidence/cli-session.cast" +MANIFEST_PATH: Final = "docs/evidence/visual-manifest.json" + +VISUAL_OUTPUT_PATHS: Final = ( + CLI_SVG_PATH, + COUNTS_SVG_PATH, + MATRIX_SVG_PATH, + PROVENANCE_SVG_PATH, + CAST_PATH, +) +ALL_OUTPUT_PATHS: Final = (*VISUAL_OUTPUT_PATHS, MANIFEST_PATH) + +VERIFICATION_SCHEMA: Final = "netveil.fresh-wheel-verification.v1" +INVENTORY_SCHEMA: Final = "netveil.release-inventory.v1" +VISUAL_MANIFEST_SCHEMA: Final = "netveil.visual-evidence.v1" +MAX_INPUT_BYTES: Final = 16 * 1_048_576 +LOWER_HEX: Final = frozenset("0123456789abcdef") +RAW_ENDPOINTS: Final = ( + "192.0.2.10", + "198.51.100.20", + "203.0.113.30", + "2001:db8::10", +) + +BACKGROUND: Final = "#07111f" +PANEL: Final = "#101d31" +PANEL_ALT: Final = "#0c1728" +BORDER: Final = "#29405e" +TEXT: Final = "#f4f8ff" +MUTED: Final = "#a8b7c9" +CYAN: Final = "#67e8f9" +VIOLET: Final = "#c4b5fd" +GREEN: Final = "#86efac" +AMBER: Final = "#fcd34d" +PINK: Final = "#fda4af" +MONO: Final = "DejaVu Sans Mono,ui-monospace,SFMono-Regular,Consolas,monospace" +SANS: Final = "DejaVu Sans,Arial,sans-serif" + + +class EvidenceRenderError(RuntimeError): + """The evidence bundle is absent, malformed, inconsistent, or stale.""" + + +@dataclass(frozen=True, slots=True) +class ArtifactFact: + """One release artifact represented in both evidence documents.""" + + filename: str + kind: str + sha256: str + size_bytes: int + member_count: int + + +@dataclass(frozen=True, slots=True) +class TraceFact: + """Path-free normalized syscall evidence.""" + + label: str + sha256: str + exec_count: int + exit_count: int + network_count: int + child_count: int + + +@dataclass(frozen=True, slots=True) +class EvidenceFacts: + """Validated facts used by every generated output.""" + + source_commit: str + source_date_epoch: int + inventory_sha256: str + artifacts: tuple[ArtifactFact, ...] + checks: tuple[str, ...] + interpreter: str + platform: str + launcher_mode: str + launcher_sha256: str + launcher_size_bytes: int + record_mode: str + record_sha256: str + record_size_bytes: int + selected_record_rows: int + traces: tuple[TraceFact, ...] + version_stdout: str + receipt: dict[str, object] + receipt_stdout: bytes + endpoint_occurrences: int + unique_endpoints: int + physical_lines: int + source_bytes: int + ipv4_occurrences: int + ipv6_occurrences: int + documentation_occurrences: int + duplicate_groups: int + + +def _fail(message: str) -> NoReturn: + raise EvidenceRenderError(message) + + +def _canonical_json(document: object) -> bytes: + return json.dumps( + document, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def _json_bytes(document: object) -> bytes: + return _canonical_json(document) + b"\n" + + +def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + output: dict[str, object] = {} + for key, value in pairs: + if key in output: + _fail("JSON contains a duplicate object key") + output[key] = value + return output + + +def _load_canonical_json(payload: bytes, label: str) -> dict[str, object]: + if ( + type(payload) is not bytes + or not 0 < len(payload) <= MAX_INPUT_BYTES + or not payload.endswith(b"\n") + or payload.endswith(b"\n\n") + ): + _fail(f"{label} is not one bounded canonical JSON line") + try: + document = json.loads( + payload.decode("ascii"), + object_pairs_hook=_unique_object, + parse_constant=lambda value: _fail( + f"{label} contains invalid constant {value}" + ), + ) + except EvidenceRenderError: + raise + except (UnicodeError, json.JSONDecodeError, ValueError) as error: + raise EvidenceRenderError(f"{label} is not valid ASCII JSON") from error + if not isinstance(document, dict) or _json_bytes(document) != payload: + _fail(f"{label} is not canonical") + return cast(dict[str, object], document) + + +def _mapping(value: object, label: str) -> dict[str, object]: + if not isinstance(value, dict): + _fail(f"{label} must be an object") + return cast(dict[str, object], value) + + +def _mapping_at( + document: Mapping[str, object], + key: str, + label: str, +) -> dict[str, object]: + if key not in document: + _fail(f"{label} is missing {key}") + return _mapping(document[key], f"{label}.{key}") + + +def _sequence(value: object, label: str) -> list[object]: + if not isinstance(value, list): + _fail(f"{label} must be an array") + return cast(list[object], value) + + +def _sequence_at( + document: Mapping[str, object], + key: str, + label: str, +) -> list[object]: + if key not in document: + _fail(f"{label} is missing {key}") + return _sequence(document[key], f"{label}.{key}") + + +def _text(value: object, label: str, *, allow_newline: bool = False) -> str: + if not isinstance(value, str) or not value.isascii(): + _fail(f"{label} must be ASCII text") + forbidden = "\x00\r" if allow_newline else "\x00\r\n" + if any(character in forbidden for character in value): + _fail(f"{label} contains a forbidden control character") + if any( + ord(character) < 32 and (allow_newline is False or character != "\n") + for character in value + ): + _fail(f"{label} contains a control character") + return value + + +def _text_at( + document: Mapping[str, object], + key: str, + label: str, + *, + allow_newline: bool = False, +) -> str: + if key not in document: + _fail(f"{label} is missing {key}") + return _text( + document[key], + f"{label}.{key}", + allow_newline=allow_newline, + ) + + +def _integer(value: object, label: str) -> int: + if type(value) is not int or value < 0: + _fail(f"{label} must be a non-negative integer") + return value + + +def _integer_at( + document: Mapping[str, object], + key: str, + label: str, +) -> int: + if key not in document: + _fail(f"{label} is missing {key}") + return _integer(document[key], f"{label}.{key}") + + +def _false_at(document: Mapping[str, object], key: str, label: str) -> None: + if document.get(key) is not False: + _fail(f"{label}.{key} must be false") + + +def _hex(value: object, label: str, length: int) -> str: + text = _text(value, label) + if len(text) != length or any(character not in LOWER_HEX for character in text): + _fail(f"{label} is not lowercase hexadecimal") + return text + + +def _mode(value: object, label: str) -> str: + text = _text(value, label) + if len(text) != 4 or any(character not in "01234567" for character in text): + _fail(f"{label} is not a four-digit mode") + return text + + +def _parse_artifacts( + artifacts: list[object], +) -> tuple[ArtifactFact, ...]: + parsed: list[ArtifactFact] = [] + for index, item in enumerate(artifacts): + artifact = _mapping(item, f"artifacts[{index}]") + filename = _text_at(artifact, "filename", "artifact") + kind = _text_at(artifact, "kind", "artifact") + if kind not in ("wheel", "sdist"): + _fail("artifact kind must be wheel or sdist") + members = ( + _sequence_at(artifact, "members", "artifact") if kind == "sdist" else [] + ) + if kind == "wheel" and "members" in artifact: + _fail("release inventory wheel must not claim sdist members") + parsed.append( + ArtifactFact( + filename=filename, + kind=kind, + sha256=_hex(artifact.get("sha256"), "artifact.sha256", 64), + size_bytes=_integer_at(artifact, "size_bytes", "artifact"), + member_count=len(members), + ) + ) + if ( + len(parsed) != 2 + or {artifact.kind for artifact in parsed} != {"wheel", "sdist"} + or [artifact.filename for artifact in parsed] + != sorted(artifact.filename for artifact in parsed) + ): + _fail("release artifacts are not the exact sorted wheel/sdist pair") + return tuple(parsed) + + +def _parse_checks(raw_checks: list[object]) -> tuple[str, ...]: + checks: list[str] = [] + for index, item in enumerate(raw_checks): + check = _mapping(item, f"checks[{index}]") + if set(check) != {"name", "status"} or check.get("status") != "pass": + _fail("every evidence check must have exact pass status") + name = _text_at(check, "name", "check") + if not name or any( + character not in "abcdefghijklmnopqrstuvwxyz_" for character in name + ): + _fail("check names must be lowercase identifiers") + checks.append(name) + required = { + "coordinated_bootstrap_record_mutation_accepted", + "public_demo_capture", + "release_inventory_integrity", + "source_commit_bound", + "syscall_trace_offline", + "tamper_fail_closed", + } + if ( + len(checks) < len(required) + or len(checks) != len(set(checks)) + or not required.issubset(checks) + ): + _fail("verification check inventory is incomplete") + return tuple(checks) + + +def _parse_traces(raw_traces: list[object]) -> tuple[TraceFact, ...]: + traces: list[TraceFact] = [] + for index, item in enumerate(raw_traces): + trace = _mapping(item, f"syscall_traces[{index}]") + chain = [ + _text(value, "trace.exec_chain item") + for value in _sequence_at(trace, "exec_chain", "trace") + ] + if chain != ["installed_launcher", "installed_python"]: + _fail("trace exec chain is not the exact launcher/Python pair") + traces.append( + TraceFact( + label=_text_at(trace, "label", "trace"), + sha256=_hex( + trace.get("normalized_sha256"), + "trace.normalized_sha256", + 64, + ), + exec_count=_integer_at(trace, "exec_count", "trace"), + exit_count=_integer_at(trace, "exit_syscall_count", "trace"), + network_count=_integer_at( + trace, + "network_syscall_count", + "trace", + ), + child_count=_integer_at( + trace, + "post_launch_process_count", + "trace", + ), + ) + ) + if {trace.label for trace in traces} != {"receipt", "version"} or any( + trace.exec_count != 2 + or trace.exit_count < 1 + or trace.network_count != 0 + or trace.child_count != 0 + for trace in traces + ): + _fail("normalized trace evidence is incomplete") + return tuple(sorted(traces, key=lambda trace: trace.label)) + + +def _parse_demo( + demo: Mapping[str, object], +) -> tuple[ + str, + dict[str, object], + bytes, + int, + int, + int, + int, + int, + int, + int, + int, +]: + if ( + demo.get("classification") + != "synthetic_ietf_documentation_ranges_with_public_demo_key" + ): + _fail("public demo classification is not exact") + corpus = _mapping_at(demo, "corpus", "public_demo") + _hex(corpus.get("sha256"), "public_demo.corpus.sha256", 64) + _integer_at(corpus, "size_bytes", "public_demo.corpus") + physical_lines = _integer_at(corpus, "physical_lines", "public_demo.corpus") + key = _mapping_at(demo, "public_demo_key", "public_demo") + if ( + key.get("classification") != "public_non_secret_test_material" + or key.get("source_constant") != "tools/verify_fresh_wheel.py:_PUBLIC_DEMO_KEY" + or _integer_at(key, "size_bytes", "public_demo.public_demo_key") != 32 + ): + _fail("public demo key classification is incomplete") + _hex(key.get("sha256"), "public_demo.public_demo_key.sha256", 64) + + commands = _sequence_at(demo, "commands", "public_demo") + if len(commands) != 2: + _fail("public demo must contain exactly two commands") + version = _mapping(commands[0], "public_demo.commands[0]") + receipt_command = _mapping(commands[1], "public_demo.commands[1]") + expected_version_argv = ["netveil-audit", "--version"] + expected_receipt_argv = [ + "netveil-audit", + "receipt", + "documentation-corpus.txt", + "--key-file", + "public-demo.key", + ] + for command, expected_argv in ( + (version, expected_version_argv), + (receipt_command, expected_receipt_argv), + ): + argv = [ + _text(value, "public demo argv item") + for value in _sequence_at(command, "argv", "public demo command") + ] + if ( + argv != expected_argv + or _integer_at(command, "exit_code", "public demo command") != 0 + or _text_at( + command, + "stderr", + "public demo command", + allow_newline=True, + ) + != "" + ): + _fail("public demo command contract changed") + version_stdout = _text_at( + version, + "stdout", + "public demo version", + allow_newline=True, + ) + if version_stdout != "netveil-audit 0.3.0\n": + _fail("public demo version output changed") + + receipt = _mapping_at(receipt_command, "stdout_json", "public demo receipt") + receipt_stdout = _json_bytes(receipt) + if hashlib.sha256(receipt_stdout).hexdigest() != _hex( + receipt_command.get("stdout_sha256"), + "public demo receipt stdout_sha256", + 64, + ): + _fail("public demo receipt stdout digest does not match") + if receipt.get("schema") != "netveil.aggregate-receipt.v1": + _fail("public demo receipt schema changed") + report = _mapping_at(receipt, "report", "public demo receipt") + counts = _mapping_at(report, "counts", "public demo report") + by_version = _mapping_at( + report, + "endpoint_occurrences_by_ip_version", + "public demo report", + ) + by_scope = _mapping_at( + report, + "endpoint_occurrences_by_scope", + "public demo report", + ) + duplicates = _mapping_at(report, "duplicates", "public demo report") + return ( + version_stdout, + receipt, + receipt_stdout, + _integer_at(counts, "endpoint_occurrences", "receipt counts"), + _integer_at(counts, "unique_endpoints", "receipt counts"), + physical_lines, + _integer_at(counts, "source_bytes", "receipt counts"), + _integer_at(by_version, "ipv4", "receipt by version"), + _integer_at(by_version, "ipv6", "receipt by version"), + _integer_at(by_scope, "documentation", "receipt by scope"), + _integer_at(duplicates, "group_count", "receipt duplicates"), + ) + + +def parse_evidence( + verification_payload: bytes, + inventory_payload: bytes, +) -> EvidenceFacts: + """Validate the cross-document evidence contract and return renderable facts.""" + + verification = _load_canonical_json( + verification_payload, + "fresh-wheel verification", + ) + inventory = _load_canonical_json(inventory_payload, "release inventory") + if ( + verification.get("schema") != VERIFICATION_SCHEMA + or verification.get("status") != "pass" + or inventory.get("schema") != INVENTORY_SCHEMA + ): + _fail("evidence schema or status is not publishable") + source_commit = _hex( + verification.get("source_commit"), + "verification.source_commit", + 40, + ) + if inventory.get("source_commit") != source_commit: + _fail("verification and inventory source commits differ") + source_date_epoch = _integer_at( + inventory, + "source_date_epoch", + "release inventory", + ) + inventory_artifacts = _sequence_at( + inventory, + "artifacts", + "release inventory", + ) + artifacts = _parse_artifacts(inventory_artifacts) + + integrity = _mapping_at( + verification, + "integrity_evidence", + "verification", + ) + if ( + integrity.get("artifacts") != inventory_artifacts + or integrity.get("inventory_schema") != INVENTORY_SCHEMA + or integrity.get("inventory_type") != "unsigned_sha256_manifest" + or integrity.get("source_commit") != source_commit + or integrity.get("source_date_epoch") != source_date_epoch + ): + _fail("verification does not embed the exact release inventory facts") + _false_at(integrity, "signature_verified", "integrity evidence") + _false_at(integrity, "attestation_verified", "integrity evidence") + inventory_sha256 = _hex( + integrity.get("inventory_sha256"), + "integrity_evidence.inventory_sha256", + 64, + ) + if inventory_sha256 != hashlib.sha256(inventory_payload).hexdigest(): + _fail("release inventory digest does not match its bytes") + + wheel = _mapping_at(verification, "wheel", "verification") + wheel_artifact = next( + artifact for artifact in artifacts if artifact.kind == "wheel" + ) + if ( + wheel.get("sha256") != wheel_artifact.sha256 + or wheel.get("size_bytes") != wheel_artifact.size_bytes + or not _sequence_at(wheel, "members", "verification wheel") + ): + _fail("wheel verification facts do not match the release inventory") + + installed = _mapping_at(verification, "installed", "verification") + launcher = _mapping_at(installed, "launcher", "installed") + record = _mapping_at(installed, "record", "installed") + rows = _sequence_at(installed, "selected_record_rows", "installed") + if not rows: + _fail("installed RECORD evidence is empty") + + interpreter = _mapping_at(verification, "interpreter", "verification") + platform = _mapping_at(verification, "platform", "verification") + demo_values = _parse_demo(_mapping_at(verification, "public_demo", "verification")) + ( + version_stdout, + receipt, + receipt_stdout, + endpoint_occurrences, + unique_endpoints, + physical_lines, + source_bytes, + ipv4_occurrences, + ipv6_occurrences, + documentation_occurrences, + duplicate_groups, + ) = demo_values + + if any( + endpoint.encode("ascii") in verification_payload for endpoint in RAW_ENDPOINTS + ): + _fail("public evidence unexpectedly contains raw endpoint text") + + return EvidenceFacts( + source_commit=source_commit, + source_date_epoch=source_date_epoch, + inventory_sha256=inventory_sha256, + artifacts=artifacts, + checks=_parse_checks(_sequence_at(verification, "checks", "verification")), + interpreter=( + f"{_text_at(interpreter, 'implementation', 'interpreter')} " + f"{_text_at(interpreter, 'version', 'interpreter')} · " + f"{_text_at(interpreter, 'cache_tag', 'interpreter')}" + ), + platform=( + f"{_text_at(platform, 'system', 'platform')} " + f"{_text_at(platform, 'release', 'platform')} · " + f"{_text_at(platform, 'machine', 'platform')}" + ), + launcher_mode=_mode(launcher.get("mode"), "installed launcher mode"), + launcher_sha256=_hex( + launcher.get("sha256"), + "installed launcher sha256", + 64, + ), + launcher_size_bytes=_integer_at( + launcher, + "size_bytes", + "installed launcher", + ), + record_mode=_mode(record.get("mode"), "installed RECORD mode"), + record_sha256=_hex( + record.get("sha256"), + "installed RECORD sha256", + 64, + ), + record_size_bytes=_integer_at(record, "size_bytes", "installed RECORD"), + selected_record_rows=len(rows), + traces=_parse_traces( + _sequence_at(verification, "syscall_traces", "verification") + ), + version_stdout=version_stdout, + receipt=receipt, + receipt_stdout=receipt_stdout, + endpoint_occurrences=endpoint_occurrences, + unique_endpoints=unique_endpoints, + physical_lines=physical_lines, + source_bytes=source_bytes, + ipv4_occurrences=ipv4_occurrences, + ipv6_occurrences=ipv6_occurrences, + documentation_occurrences=documentation_occurrences, + duplicate_groups=duplicate_groups, + ) + + +def _human_bytes(value: int) -> str: + if value < 1_024: + return f"{value} B" + if value < 1_048_576: + return f"{value / 1_024:.1f} KiB" + return f"{value / 1_048_576:.1f} MiB" + + +def _svg_document( + *, + title: str, + description: str, + width: int, + height: int, + body: Sequence[str], +) -> bytes: + escaped_title = html.escape(title) + escaped_description = html.escape(description) + lines = [ + ( + f'' + ), + f'{escaped_title}', + f'{escaped_description}', + "", + f'', + *body, + "", + ] + return ("\n".join(lines) + "\n").encode("utf-8") + + +def _terminal_transcript(facts: EvidenceFacts) -> str: + receipt = facts.receipt_stdout.decode("ascii").rstrip("\n") + return ( + "Netveil installed-wheel public demonstration\n" + f"source commit: {facts.source_commit}\n" + "input class: synthetic IETF documentation ranges\n" + "key class: public non-secret test material\n\n" + "$ netveil-audit --version\n" + f"{facts.version_stdout}" + "$ netveil-audit receipt documentation-corpus.txt " + "--key-file public-demo.key\n" + f"{receipt}\n" + ) + + +def _render_cli(facts: EvidenceFacts) -> bytes: + wrapped: list[str] = [] + for line in _terminal_transcript(facts).rstrip("\n").splitlines(): + wrapped.extend( + textwrap.wrap( + line, + width=126, + subsequent_indent=" ", + break_long_words=True, + break_on_hyphens=False, + replace_whitespace=False, + drop_whitespace=False, + ) + or [""] + ) + line_height = 21 + height = 245 + len(wrapped) * line_height + body = [ + 'Installed CLI · captured stdout', + ( + 'Exact successful output ' + "captured by the fresh-wheel verifier; long canonical JSON is " + "visually wrapped only." + ), + ( + f'' + ), + '', + '', + '', + ] + y = 180 + for line in wrapped: + fill = CYAN if line.startswith("$ ") else TEXT + body.append( + f'' + f"{html.escape(line)}" + ) + y += line_height + return _svg_document( + title="Netveil installed CLI evidence", + description=( + "Terminal rendering of exact version and receipt standard output " + "captured from the verified fresh wheel." + ), + width=1440, + height=height, + body=body, + ) + + +def _render_counts(facts: EvidenceFacts) -> bytes: + metrics = ( + ("physical lines", facts.physical_lines, CYAN), + ("endpoint occurrences", facts.endpoint_occurrences, VIOLET), + ("unique endpoints", facts.unique_endpoints, GREEN), + ("IPv4 occurrences", facts.ipv4_occurrences, "#38bdf8"), + ("IPv6 occurrences", facts.ipv6_occurrences, "#f472b6"), + ("duplicate groups", facts.duplicate_groups, AMBER), + ) + maximum = max(value for _, value, _ in metrics) + body = [ + 'Public receipt · disclosed counts', + ( + 'Values parsed from the exact ' + "verifier-captured receipt; identifiers and raw endpoints are not " + "used in this chart." + ), + '', + ] + y = 175 + for label, value, color in metrics: + width = 820 * value / maximum if maximum else 0 + body.extend( + ( + f'{html.escape(label)}', + f'', + f'', + f'{value}', + ) + ) + y += 62 + body.extend( + ( + '', + 'Scope result', + ( + f'{facts.documentation_occurrences} / ' + f"{facts.endpoint_occurrences} occurrences · documentation ranges" + ), + ( + 'Synthetic corpus only; this ' + "is workflow evidence, not a claim about live infrastructure." + ), + '', + 'Disclosure boundary', + ( + f'{facts.source_bytes} source bytes ' + "are summarized, not embedded" + ), + ( + 'Counts, equality and ' + "frequency remain public; pseudonymized does not mean anonymous." + ), + ) + ) + return _svg_document( + title="Netveil public receipt count evidence", + description=( + "Bar chart of physical lines, endpoint counts, IP versions, and " + "duplicate groups parsed from the committed public demo receipt." + ), + width=1440, + height=830, + body=body, + ) + + +def _render_matrix(facts: EvidenceFacts) -> bytes: + columns = 3 + rows = (len(facts.checks) + columns - 1) // columns + height = 500 + rows * 58 + body = [ + 'Fresh-wheel verification matrix', + ( + f'{len(facts.checks)} exact ' + "checks reported pass by the committed verifier JSON." + ), + '', + (f'{html.escape(facts.interpreter)}'), + f'{html.escape(facts.platform)}', + ( + f'' + f"{facts.source_commit}" + ), + ( + '' + "unsigned consistency evidence · not attestation" + ), + ] + for index, check in enumerate(facts.checks): + column = index % columns + row = index // columns + x = 48 + column * 448 + y = 270 + row * 58 + body.extend( + ( + f'', + f'', + ( + f'' + f"{html.escape(check.replace('_', ' '))}" + ), + f'PASS', + ) + ) + trace_y = 300 + rows * 58 + body.append( + f'Normalized offline syscall evidence' + ) + for index, trace in enumerate(facts.traces): + x = 48 + index * 680 + body.extend( + ( + f'', + ( + f'' + f"{html.escape(trace.label)}" + ), + ( + f'' + f"exec={trace.exec_count} · network={trace.network_count} · " + f"post-launch children={trace.child_count}" + ), + ( + f'' + f"normalized sha256 {trace.sha256}" + ), + ) + ) + return _svg_document( + title="Netveil fresh-wheel verification matrix", + description=( + "All verifier pass labels, interpreter and platform identity, and " + "normalized path-free process and network trace evidence." + ), + width=1440, + height=height, + body=body, + ) + + +def _render_provenance(facts: EvidenceFacts) -> bytes: + wheel = next(artifact for artifact in facts.artifacts if artifact.kind == "wheel") + sdist = next(artifact for artifact in facts.artifacts if artifact.kind == "sdist") + boxes = ( + ( + 48, + "1 · clean Git HEAD", + facts.source_commit, + f"SOURCE_DATE_EPOCH {facts.source_date_epoch}", + CYAN, + ), + ( + 390, + "2 · bound release artifacts", + f"wheel · {_human_bytes(wheel.size_bytes)} · {wheel.sha256[:20]}…", + ( + f"sdist · {_human_bytes(sdist.size_bytes)} · " + f"{sdist.member_count} members" + ), + VIOLET, + ), + ( + 732, + "3 · fresh installed state", + ( + f"launcher {facts.launcher_mode} · " + f"{_human_bytes(facts.launcher_size_bytes)}" + ), + ( + f"RECORD {facts.record_mode} · " + f"{facts.selected_record_rows} selected rows" + ), + GREEN, + ), + ( + 1074, + "4 · executed evidence", + f"{len(facts.checks)} checks pass", + "2 traces · network 0 · children 0", + AMBER, + ), + ) + body = [ + 'Release evidence chain', + ( + 'Every displayed value comes ' + "from the canonical release inventory or fresh-wheel verifier JSON." + ), + ] + for index, (x, title, line_one, line_two, color) in enumerate(boxes): + body.extend( + ( + f'', + f'', + f'{html.escape(title)}', + f'{html.escape(line_one)}', + f'{html.escape(line_two)}', + ( + f'BOUND IN EVIDENCE' + if index + else f'BUILDER-OBSERVED' + ), + ) + ) + if index < len(boxes) - 1: + body.extend( + ( + f'', + f'', + ) + ) + body.extend( + ( + '', + 'Exact installed bindings', + ( + f'launcher sha256 ' + f"{facts.launcher_sha256}" + ), + ( + f'RECORD sha256 ' + f"{facts.record_sha256}" + ), + ( + f'inventory sha256 ' + f"{facts.inventory_sha256}" + ), + '', + 'Claim boundary', + ( + 'The manifest is unsigned. ' + "It shows internally consistent bytes and an executed test run; " + "it does not prove publisher identity, an honest host, or an " + "unmodified verifier." + ), + ( + 'Authenticate the published ' + "artifact digests through a separate trusted channel." + ), + ) + ) + return _svg_document( + title="Netveil release evidence chain", + description=( + "Source commit, artifact inventory, installed-file, and " + "fresh execution evidence with an explicit unsigned non-claim." + ), + width=1440, + height=825, + body=body, + ) + + +def _render_cast(facts: EvidenceFacts) -> bytes: + command_one = "$ netveil-audit --version\r\n" + command_two = ( + "$ netveil-audit receipt documentation-corpus.txt " + "--key-file public-demo.key\r\n" + ) + receipt = facts.receipt_stdout.decode("ascii").replace("\n", "\r\n") + header = { + "env": {"SHELL": "/bin/sh", "TERM": "xterm-256color"}, + "height": 40, + "timestamp": facts.source_date_epoch, + "version": 2, + "width": 120, + } + events: tuple[object, ...] = ( + [ + 0.1, + "o", + ( + "Netveil verified public demo\r\n" + "synthetic IETF documentation ranges · public non-secret key\r\n\r\n" + ), + ], + [0.6, "o", command_one], + [0.9, "o", facts.version_stdout.replace("\n", "\r\n")], + [1.4, "o", command_two], + [1.8, "o", receipt], + ) + lines = [_canonical_json(header), *(_canonical_json(event) for event in events)] + return b"\n".join(lines) + b"\n" + + +def render_bundle( + verification_payload: bytes, + inventory_payload: bytes, + *, + generator_payload: bytes, +) -> dict[str, bytes]: + """Return every generated output, including the non-self-referential manifest.""" + + facts = parse_evidence(verification_payload, inventory_payload) + outputs = { + CLI_SVG_PATH: _render_cli(facts), + COUNTS_SVG_PATH: _render_counts(facts), + MATRIX_SVG_PATH: _render_matrix(facts), + PROVENANCE_SVG_PATH: _render_provenance(facts), + CAST_PATH: _render_cast(facts), + } + manifest = { + "claim_boundary": ( + "unsigned internally consistent build-and-execution evidence; " + "not publisher authentication, host attestation, or a signature" + ), + "generator": { + "path": "tools/render_evidence.py", + "sha256": hashlib.sha256(generator_payload).hexdigest(), + "size_bytes": len(generator_payload), + }, + "inputs": [ + { + "path": "docs/evidence/fresh-wheel-verification.json", + "sha256": hashlib.sha256(verification_payload).hexdigest(), + "size_bytes": len(verification_payload), + }, + { + "path": "docs/evidence/release-inventory.json", + "sha256": hashlib.sha256(inventory_payload).hexdigest(), + "size_bytes": len(inventory_payload), + }, + ], + "outputs": [ + { + "path": path, + "sha256": hashlib.sha256(outputs[path]).hexdigest(), + "size_bytes": len(outputs[path]), + } + for path in sorted(outputs) + ], + "schema": VISUAL_MANIFEST_SCHEMA, + "source_commit": facts.source_commit, + } + outputs[MANIFEST_PATH] = _json_bytes(manifest) + return outputs + + +def _read_input(path: Path, label: str) -> bytes: + try: + status = path.lstat() + if ( + not stat.S_ISREG(status.st_mode) + or not 0 < status.st_size <= MAX_INPUT_BYTES + ): + _fail(f"{label} is not one bounded regular file") + payload = path.read_bytes() + except OSError as error: + raise EvidenceRenderError(f"{label} cannot be read") from error + if len(payload) != status.st_size: + _fail(f"{label} changed while being read") + return payload + + +def _assert_safe_target(path: Path) -> None: + try: + relative = path.relative_to(ROOT) + except ValueError: + _fail("generated output escaped the repository") + current = ROOT + for part in relative.parts[:-1]: + current /= part + if current.exists() and current.is_symlink(): + _fail(f"generated output parent is a symlink: {relative.as_posix()}") + if path.exists(): + try: + mode = path.lstat().st_mode + except OSError as error: + raise EvidenceRenderError("generated output cannot be inspected") from error + if not stat.S_ISREG(mode): + _fail(f"generated output is not regular: {relative.as_posix()}") + + +def _replace_file(path: Path, payload: bytes) -> None: + _assert_safe_target(path) + try: + path.parent.mkdir(mode=0o755, parents=True, exist_ok=True) + descriptor, raw_temporary = tempfile.mkstemp( + prefix=f".{path.name}.", + dir=path.parent, + ) + temporary = Path(raw_temporary) + try: + os.fchmod(descriptor, 0o644) + offset = 0 + while offset < len(payload): + written = os.write(descriptor, payload[offset:]) + if written <= 0: + raise OSError("write made no progress") + offset += written + os.fsync(descriptor) + finally: + os.close(descriptor) + os.replace(temporary, path) + except OSError as error: + try: + temporary.unlink(missing_ok=True) + except (OSError, UnboundLocalError): + pass + raise EvidenceRenderError(f"cannot publish {path.name}") from error + + +def _check_outputs(outputs: Mapping[str, bytes]) -> None: + stale: list[str] = [] + for relative_path in ALL_OUTPUT_PATHS: + path = ROOT / relative_path + _assert_safe_target(path) + try: + observed = path.read_bytes() + except OSError: + stale.append(relative_path) + continue + if observed != outputs[relative_path]: + stale.append(relative_path) + if stale: + _fail("stale generated evidence: " + ", ".join(stale)) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Render deterministic SVG and terminal-cast views from committed " + "Netveil release evidence." + ) + ) + parser.add_argument( + "--check", + action="store_true", + help="fail unless every committed generated byte is current", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + namespace = _parser().parse_args(argv) + try: + verification_payload = _read_input( + VERIFICATION_PATH, + "fresh-wheel verification", + ) + inventory_payload = _read_input(INVENTORY_PATH, "release inventory") + generator_payload = _read_input(GENERATOR_PATH, "evidence renderer") + outputs = render_bundle( + verification_payload, + inventory_payload, + generator_payload=generator_payload, + ) + if namespace.check: + _check_outputs(outputs) + else: + for relative_path in VISUAL_OUTPUT_PATHS: + _replace_file(ROOT / relative_path, outputs[relative_path]) + _replace_file(ROOT / MANIFEST_PATH, outputs[MANIFEST_PATH]) + except EvidenceRenderError as error: + print(f"netveil-evidence-renderer: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/verify_fresh_wheel.py b/tools/verify_fresh_wheel.py new file mode 100755 index 0000000..9d53e56 --- /dev/null +++ b/tools/verify_fresh_wheel.py @@ -0,0 +1,2472 @@ +#!/usr/bin/env python3 +"""Verify one pinned Netveil wheel in a disposable, offline virtual environment. + +The verifier never builds a distribution and never resolves dependencies. It +first pins and inspects the exact wheel bytes supplied by the caller, then asks +pip to install a private copy with ``--no-index --no-deps``. Product processes +run with bounded output capture and receive only synthetic IETF documentation +range inputs. +""" + +from __future__ import annotations + +import base64 +import csv +import gzip +import hashlib +import importlib.metadata +import importlib.util +import io +import json +import marshal +import os +import re +import shutil +import signal +import stat +import struct +import subprocess +import sys +import tarfile +import tempfile +import venv +import zlib +from collections.abc import Iterator, Mapping, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from email import policy +from email.parser import BytesParser +from pathlib import Path, PurePosixPath +from types import CodeType +from typing import Final, NoReturn +from zipfile import BadZipFile, ZipFile, ZipInfo + +_DISTRIBUTION_NAME: Final = "netveil-audit" +_DISTRIBUTION_VERSION: Final = "0.3.0" +_WHEEL_NAME: Final = "netveil_audit-0.3.0-py3-none-any.whl" +_SDIST_NAME: Final = "netveil_audit-0.3.0.tar.gz" +_DIST_INFO: Final = "netveil_audit-0.3.0.dist-info" +_WHEEL_SCRIPT: Final = "netveil_audit-0.3.0.data/scripts/netveil-audit" +_LAUNCHER_NAME: Final = "netveil-audit" +_BOOTSTRAP_NAME: Final = "netveil_bootstrap.py" +_SCHEMA: Final = "netveil.fresh-wheel-verification.v1" +_INVENTORY_SCHEMA: Final = "netveil.release-inventory.v1" +_INVENTORY_NAME: Final = "release-inventory.json" + +_MAX_WHEEL_BYTES: Final = 32 * 1_048_576 +_MAX_INVENTORY_BYTES: Final = 16 * 1_048_576 +_MAX_RELEASE_ARTIFACT_BYTES: Final = 512 * 1_048_576 +_MAX_COMPRESSED_SDIST_BYTES: Final = 128 * 1_048_576 +_MAX_EXPANDED_SDIST_BYTES: Final = 256 * 1_048_576 +_MAX_SDIST_MEMBER_BYTES: Final = 64 * 1_048_576 +_MAX_SDIST_FILE_BYTES: Final = 192 * 1_048_576 +_MAX_SDIST_MEMBERS: Final = 20_000 +_MAX_MEMBER_BYTES: Final = 2 * 1_048_576 +_MAX_EXPANDED_BYTES: Final = 16 * 1_048_576 +_MAX_PROCESS_OUTPUT_BYTES: Final = 1_048_576 +_PROCESS_TIMEOUT_SECONDS: Final = 30.0 +_ARTIFACT_FAILURE: Final = b"netveil-audit: artifact_unverified\n" +_LOWER_HEX: Final = frozenset("0123456789abcdef") +_TAR_BLOCK_BYTES: Final = 512 +_TAR_TRAILER_BYTES: Final = 2 * _TAR_BLOCK_BYTES + +_EXPECTED_WHEEL_MEMBERS: Final = frozenset( + { + _WHEEL_SCRIPT, + "netveil/__init__.py", + "netveil/cli.py", + "netveil/model.py", + "netveil/parser.py", + "netveil/privacy.py", + "netveil/py.typed", + _BOOTSTRAP_NAME, + f"{_DIST_INFO}/METADATA", + f"{_DIST_INFO}/RECORD", + f"{_DIST_INFO}/WHEEL", + f"{_DIST_INFO}/licenses/LICENSE", + f"{_DIST_INFO}/top_level.txt", + } +) +_CHECKS: Final = ( + "wheel_archive", + "fresh_install", + "entry_points_absent", + "launcher_identity", + "direct_and_path_commands", + "isolated_startup", + "environment_injection_inert", + "unchecked_bytecode_inert", + "unknown_bytecode_rejected", + "tamper_fail_closed", + "metadata_drift_rejected", + "coordinated_bootstrap_record_mutation_accepted", + "unknown_package_file_rejected", + "receipt_deterministic", + "receipt_redacted", + "public_demo_capture", + "syscall_trace_offline", + "release_inventory_integrity", + "source_commit_bound", +) +_CORPUS: Final = ( + b"# Synthetic IETF documentation ranges only\n" + b"192.0.2.10:443\n" + b"192.0.2.10:443\n" + b"198.51.100.20:80\n" + b"203.0.113.30:65535\n" + b"[2001:db8::10]:8443\n" +) +# Deliberately public, deterministic demonstration material. It is never +# suitable for a private corpus; its sole purpose is reproducible CLI output +# over the synthetic IETF documentation-range corpus above. +_PUBLIC_DEMO_KEY: Final = b"netveil-public-demo-key-v1-00001" +_RAW_ENDPOINT_TOKENS: Final = ( + b"192.0.2.10", + b"198.51.100.20", + b"203.0.113.30", + b"2001:db8::10", +) +_ATTACK_SOURCE: Final = ( + b"import os\n" + b"from pathlib import Path\n" + b'Path(os.environ["NETVEIL_VERIFY_MARKER"]).write_bytes(b"executed")\n' +) +_PROBE_SOURCE: Final = b"""\ +import json +import sys +from pathlib import Path + +def main(argv): + root = str(Path(__file__).parent) + document = { + "argv_exact": list(argv) == ["isolation-probe"], + "dont_write_bytecode": sys.flags.dont_write_bytecode, + "empty_path_absent": "" not in sys.path, + "ignore_environment": sys.flags.ignore_environment, + "implementation": sys.implementation.name, + "isolated": sys.flags.isolated, + "netveil_package_absent": not any( + name == "netveil" or name.startswith("netveil.") + for name in sys.modules + ), + "no_site": sys.flags.no_site, + "safe_path": sys.flags.safe_path, + "cache_tag": sys.implementation.cache_tag, + "site_root_count": sys.path.count(root), + "version": "{}.{}.{}".format(*sys.version_info[:3]), + } + sys.stdout.write( + json.dumps(document, sort_keys=True, separators=(",", ":")) + "\\n" + ) + sys.stdout.flush() + return 0 +""" +_EXPECTED_PROBE: Final = { + "argv_exact": True, + "dont_write_bytecode": 1, + "empty_path_absent": True, + "ignore_environment": 1, + "netveil_package_absent": True, + "isolated": 1, + "no_site": 1, + "safe_path": True, + "site_root_count": 1, +} + + +class VerificationFailure(RuntimeError): + """A stable, deliberately redacted verification failure.""" + + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +@dataclass(frozen=True, slots=True) +class RecordEntry: + """One parsed wheel or installed RECORD row.""" + + path: str + digest: str | None + size: int | None + + +@dataclass(frozen=True, slots=True) +class WheelMemberEvidence: + """One archive member bound into the public verification evidence.""" + + path: str + sha256: str + size: int + mode: str + + +@dataclass(frozen=True, slots=True) +class WheelEvidence: + """Pinned evidence extracted without installing the wheel.""" + + payload: bytes + sha256: str + launcher: bytes + members: tuple[WheelMemberEvidence, ...] + + +@dataclass(frozen=True, slots=True) +class InventoryArtifactEvidence: + """One artifact digest asserted by the unsigned builder inventory.""" + + filename: str + kind: str + sha256: str + size_bytes: int + members: tuple[InventoryMemberEvidence, ...] | None + + +@dataclass(frozen=True, slots=True) +class InventoryMemberEvidence: + """One safe logical member of the canonical release sdist.""" + + path: str + kind: str + mode: str + size_bytes: int + sha256: str | None + + +@dataclass(frozen=True, slots=True) +class ReleaseInventoryEvidence: + """Strictly parsed unsigned integrity inventory from the release builder.""" + + source_commit: str + source_date_epoch: int + sha256: str + artifacts: tuple[InventoryArtifactEvidence, ...] + + +@dataclass(frozen=True, slots=True) +class ProcessResult: + """Bounded child-process result.""" + + returncode: int + stdout: bytes + stderr: bytes + + +@dataclass(frozen=True, slots=True) +class InstalledLayout: + """Paths inside one disposable installation.""" + + prefix: Path + python: Path + launcher: Path + site_root: Path + dist_info: Path + bootstrap: Path + package_root: Path + record: Path + evidence: InstalledEvidence | None = None + + +@dataclass(frozen=True, slots=True) +class InstalledRecordRowEvidence: + """One selected installed RECORD binding with no private filesystem path.""" + + path: str + sha256: str | None + size_bytes: int | None + + +@dataclass(frozen=True, slots=True) +class InstalledFileEvidence: + """Stable content and mode evidence for one installed regular file.""" + + logical_path: str + mode: str + sha256: str + size_bytes: int + + +@dataclass(frozen=True, slots=True) +class InstalledEvidence: + """Path-free evidence captured from the fresh installed artifact.""" + + launcher: InstalledFileEvidence + record: InstalledFileEvidence + selected_record_rows: tuple[InstalledRecordRowEvidence, ...] + + +@dataclass(frozen=True, slots=True) +class InterpreterEvidence: + """Path-free identity reported by the isolated installed interpreter.""" + + implementation: str + version: str + cache_tag: str + + +@dataclass(frozen=True, slots=True) +class PlatformEvidence: + """Hostname-free verifier platform identity.""" + + sys_platform: str + system: str + release: str + machine: str + + +@dataclass(frozen=True, slots=True) +class TraceEvidence: + """Normalized path-free evidence from one process/network syscall trace.""" + + label: str + normalized_sha256: str + process_count: int + exec_chain: tuple[str, ...] + exec_count: int + exit_syscall_count: int + network_syscall_count: int + post_launch_process_count: int + + +@dataclass(frozen=True, slots=True) +class ReceiptEvidence: + """Private paths retained only for the syscall trace gate.""" + + corpus: Path + key: Path + output: bytes + + +@dataclass(frozen=True, slots=True) +class PublicDemoEvidence: + """Reproducible public CLI output over synthetic, non-secret inputs.""" + + corpus_sha256: str + corpus_size_bytes: int + corpus_physical_lines: int + public_key_sha256: str + public_key_size_bytes: int + version_stdout: str + receipt: dict[str, object] + receipt_stdout_sha256: str + + +@dataclass(frozen=True, slots=True) +class VerificationSummary: + """Stable evidence safe to render outside the private workspace.""" + + source_commit: str + release_inventory: ReleaseInventoryEvidence + installed: InstalledEvidence + interpreter: InterpreterEvidence + platform: PlatformEvidence + syscall_traces: tuple[TraceEvidence, ...] + public_demo: PublicDemoEvidence + wheel_sha256: str + wheel_size_bytes: int + wheel_members: tuple[WheelMemberEvidence, ...] + + def document(self) -> dict[str, object]: + artifact_documents: list[dict[str, object]] = [] + for artifact in self.release_inventory.artifacts: + artifact_document: dict[str, object] = { + "filename": artifact.filename, + "kind": artifact.kind, + "sha256": artifact.sha256, + "size_bytes": artifact.size_bytes, + } + if artifact.members is not None: + member_documents: list[dict[str, object]] = [] + for member in artifact.members: + member_document: dict[str, object] = { + "kind": member.kind, + "mode": member.mode, + "path": member.path, + "size_bytes": member.size_bytes, + } + if member.sha256 is not None: + member_document["sha256"] = member.sha256 + member_documents.append(member_document) + artifact_document["members"] = member_documents + artifact_documents.append(artifact_document) + + def installed_file_document( + evidence: InstalledFileEvidence, + ) -> dict[str, object]: + return { + "logical_path": evidence.logical_path, + "mode": evidence.mode, + "sha256": evidence.sha256, + "size_bytes": evidence.size_bytes, + } + + return { + "checks": [{"name": name, "status": "pass"} for name in _CHECKS], + "installed": { + "launcher": installed_file_document(self.installed.launcher), + "record": installed_file_document(self.installed.record), + "selected_record_rows": [ + { + "path": row.path, + "sha256": row.sha256, + "size_bytes": row.size_bytes, + } + for row in self.installed.selected_record_rows + ], + }, + "integrity_evidence": { + "artifacts": artifact_documents, + "attestation_verified": False, + "inventory_schema": _INVENTORY_SCHEMA, + "inventory_sha256": self.release_inventory.sha256, + "inventory_type": "unsigned_sha256_manifest", + "signature_verified": False, + "source_commit": self.release_inventory.source_commit, + "source_date_epoch": self.release_inventory.source_date_epoch, + }, + "interpreter": { + "cache_tag": self.interpreter.cache_tag, + "implementation": self.interpreter.implementation, + "version": self.interpreter.version, + }, + "platform": { + "machine": self.platform.machine, + "release": self.platform.release, + "sys_platform": self.platform.sys_platform, + "system": self.platform.system, + }, + "public_demo": { + "classification": ( + "synthetic_ietf_documentation_ranges_with_public_demo_key" + ), + "commands": [ + { + "argv": ["netveil-audit", "--version"], + "exit_code": 0, + "stderr": "", + "stdout": self.public_demo.version_stdout, + }, + { + "argv": [ + "netveil-audit", + "receipt", + "documentation-corpus.txt", + "--key-file", + "public-demo.key", + ], + "exit_code": 0, + "stderr": "", + "stdout_json": self.public_demo.receipt, + "stdout_sha256": self.public_demo.receipt_stdout_sha256, + }, + ], + "corpus": { + "physical_lines": self.public_demo.corpus_physical_lines, + "sha256": self.public_demo.corpus_sha256, + "size_bytes": self.public_demo.corpus_size_bytes, + }, + "public_demo_key": { + "classification": "public_non_secret_test_material", + "sha256": self.public_demo.public_key_sha256, + "size_bytes": self.public_demo.public_key_size_bytes, + "source_constant": ("tools/verify_fresh_wheel.py:_PUBLIC_DEMO_KEY"), + }, + }, + "schema": _SCHEMA, + "source_commit": self.source_commit, + "status": "pass", + "syscall_traces": [ + { + "exec_chain": list(trace.exec_chain), + "exec_count": trace.exec_count, + "exit_syscall_count": trace.exit_syscall_count, + "label": trace.label, + "network_syscall_count": trace.network_syscall_count, + "normalized_sha256": trace.normalized_sha256, + "post_launch_process_count": trace.post_launch_process_count, + "process_count": trace.process_count, + } + for trace in self.syscall_traces + ], + "wheel": { + "members": [ + { + "mode": member.mode, + "path": member.path, + "sha256": member.sha256, + "size_bytes": member.size, + } + for member in self.wheel_members + ], + "sha256": self.wheel_sha256, + "size_bytes": self.wheel_size_bytes, + }, + } + + +def _fail(code: str) -> NoReturn: + raise VerificationFailure(code) + + +def _identity(value: os.stat_result) -> tuple[int, ...]: + return ( + value.st_dev, + value.st_ino, + value.st_mode, + value.st_uid, + value.st_gid, + value.st_nlink, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + + +def _read_exact_inventory(path: Path) -> bytes: + if os.name != "posix" or path.name != _INVENTORY_NAME: + _fail("inventory_path_invalid") + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if any(not hasattr(os, name) for name in required): + _fail("platform_unsupported") + flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + descriptor = -1 + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size <= 0 + or before.st_size > _MAX_INVENTORY_BYTES + ): + _fail("inventory_path_invalid") + chunks: list[bytes] = [] + observed = 0 + while observed <= before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail("inventory_path_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("inventory_path_invalid") + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail("inventory_path_changed") + return payload + + +def _reject_json_constant(_: str) -> NoReturn: + _fail("inventory_invalid") + + +def _unique_json_object( + pairs: list[tuple[str, object]], +) -> dict[str, object]: + document: dict[str, object] = {} + for key, value in pairs: + if key in document: + _fail("inventory_invalid") + document[key] = value + return document + + +def _parse_inventory_members( + raw_members: object, +) -> tuple[InventoryMemberEvidence, ...]: + if ( + not isinstance(raw_members, list) + or not 1 <= len(raw_members) <= _MAX_SDIST_MEMBERS + ): + _fail("inventory_invalid") + members: list[InventoryMemberEvidence] = [] + total_file_bytes = 0 + for raw_member in raw_members: + if not isinstance(raw_member, dict): + _fail("inventory_invalid") + kind = raw_member.get("kind") + expected_keys = ( + {"kind", "mode", "path", "size_bytes", "sha256"} + if kind == "file" + else {"kind", "mode", "path", "size_bytes"} + ) + if set(raw_member) != expected_keys: + _fail("inventory_invalid") + path_text = raw_member["path"] + mode = raw_member["mode"] + size_bytes = raw_member["size_bytes"] + sha256 = raw_member.get("sha256") + if not isinstance(path_text, str): + _fail("inventory_invalid") + path = PurePosixPath(path_text) + first = path.parts[0] if path.parts else "" + if ( + not path_text + or "\\" in path_text + or path.is_absolute() + or path.as_posix() != path_text + or any(part in ("", ".", "..") for part in path.parts) + or (len(first) >= 2 and first[0].isalpha() and first[1] == ":") + or any( + ord(character) < 32 or ord(character) == 127 for character in path_text + ) + or kind not in ("directory", "file") + or not isinstance(mode, str) + or type(size_bytes) is not int + ): + _fail("inventory_invalid") + if kind == "directory": + if mode != "0755" or size_bytes != 0 or sha256 is not None: + _fail("inventory_invalid") + else: + if ( + mode not in ("0644", "0755") + or not 0 <= size_bytes <= _MAX_SDIST_MEMBER_BYTES + or not isinstance(sha256, str) + or len(sha256) != 64 + or any(character not in _LOWER_HEX for character in sha256) + ): + _fail("inventory_invalid") + total_file_bytes += size_bytes + if total_file_bytes > _MAX_SDIST_FILE_BYTES: + _fail("inventory_invalid") + members.append( + InventoryMemberEvidence( + path=path_text, + kind=kind, + mode=mode, + size_bytes=size_bytes, + sha256=sha256, + ) + ) + paths = [member.path for member in members] + if paths != sorted(paths) or len(paths) != len(set(paths)): + _fail("inventory_invalid") + return tuple(members) + + +def _parse_release_inventory(payload: bytes) -> ReleaseInventoryEvidence: + if ( + type(payload) is not bytes + or not payload.endswith(b"\n") + or payload.endswith(b"\n\n") + ): + _fail("inventory_not_canonical") + try: + text = payload.decode("ascii") + document = json.loads( + text, + object_pairs_hook=_unique_json_object, + parse_constant=_reject_json_constant, + ) + except VerificationFailure: + raise + except (UnicodeError, json.JSONDecodeError, ValueError): + _fail("inventory_invalid") + if _canonical_json(document) + b"\n" != payload: + _fail("inventory_not_canonical") + if ( + not isinstance(document, dict) + or set(document) + != { + "artifacts", + "schema", + "source_commit", + "source_date_epoch", + } + or document["schema"] != _INVENTORY_SCHEMA + ): + _fail("inventory_invalid") + source_commit = document["source_commit"] + if not isinstance(source_commit, str): + _fail("inventory_invalid") + try: + _validate_source_commit(source_commit) + except VerificationFailure: + _fail("inventory_invalid") + source_date_epoch = document["source_date_epoch"] + if ( + type(source_date_epoch) is not int + or not 0 <= source_date_epoch <= (1 << 32) - 1 + ): + _fail("inventory_invalid") + raw_artifacts = document["artifacts"] + if not isinstance(raw_artifacts, list) or len(raw_artifacts) != 2: + _fail("inventory_invalid") + artifacts: list[InventoryArtifactEvidence] = [] + for raw_artifact in raw_artifacts: + if not isinstance(raw_artifact, dict): + _fail("inventory_invalid") + kind = raw_artifact.get("kind") + expected_keys = {"filename", "kind", "sha256", "size_bytes"} + if kind == "sdist": + expected_keys.add("members") + if set(raw_artifact) != expected_keys: + _fail("inventory_invalid") + filename = raw_artifact["filename"] + sha256 = raw_artifact["sha256"] + size_bytes = raw_artifact["size_bytes"] + if ( + not isinstance(filename, str) + or not filename + or "\\" in filename + or PurePosixPath(filename).name != filename + or any( + ord(character) < 32 or ord(character) == 127 for character in filename + ) + or kind not in ("wheel", "sdist") + or not isinstance(sha256, str) + or len(sha256) != 64 + or any(character not in _LOWER_HEX for character in sha256) + or type(size_bytes) is not int + or not 0 < size_bytes <= _MAX_RELEASE_ARTIFACT_BYTES + ): + _fail("inventory_invalid") + members = ( + _parse_inventory_members(raw_artifact["members"]) + if kind == "sdist" + else None + ) + artifacts.append( + InventoryArtifactEvidence( + filename=filename, + kind=kind, + sha256=sha256, + size_bytes=size_bytes, + members=members, + ) + ) + if [artifact.filename for artifact in artifacts] != sorted( + artifact.filename for artifact in artifacts + ) or {artifact.kind for artifact in artifacts} != {"wheel", "sdist"}: + _fail("inventory_invalid") + return ReleaseInventoryEvidence( + source_commit=source_commit, + source_date_epoch=source_date_epoch, + sha256=hashlib.sha256(payload).hexdigest(), + artifacts=tuple(artifacts), + ) + + +def _bind_release_inventory( + inventory: ReleaseInventoryEvidence, + *, + expected_source_commit: str, + wheel_path: Path, + wheel_payload: bytes, + sdist_path: Path, + sdist_payload: bytes, +) -> None: + if inventory.source_commit != expected_source_commit: + _fail("inventory_source_commit_mismatch") + by_kind = {artifact.kind: artifact for artifact in inventory.artifacts} + wheel = by_kind["wheel"] + sdist = by_kind["sdist"] + if wheel.filename != _WHEEL_NAME or wheel_path.name != wheel.filename: + _fail("inventory_wheel_name_mismatch") + if sdist.filename != _SDIST_NAME: + _fail("inventory_sdist_name_mismatch") + if sdist_path.name != sdist.filename: + _fail("inventory_sdist_name_mismatch") + if wheel.size_bytes != len(wheel_payload): + _fail("inventory_wheel_size_mismatch") + if wheel.sha256 != hashlib.sha256(wheel_payload).hexdigest(): + _fail("inventory_wheel_sha256_mismatch") + if sdist.size_bytes != len(sdist_payload): + _fail("inventory_sdist_size_mismatch") + if sdist.sha256 != hashlib.sha256(sdist_payload).hexdigest(): + _fail("inventory_sdist_sha256_mismatch") + + +def _bind_sdist_members( + inventory: ReleaseInventoryEvidence, + actual_members: tuple[InventoryMemberEvidence, ...], +) -> None: + sdist = next( + artifact for artifact in inventory.artifacts if artifact.kind == "sdist" + ) + if sdist.members != actual_members: + _fail("inventory_sdist_members_mismatch") + + +def _read_exact_wheel(path: Path) -> bytes: + if os.name != "posix" or path.name != _WHEEL_NAME: + _fail("wheel_path_invalid") + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if any(not hasattr(os, name) for name in required): + _fail("platform_unsupported") + flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + descriptor = -1 + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size <= 0 + or before.st_size > _MAX_WHEEL_BYTES + ): + _fail("wheel_path_invalid") + chunks: list[bytes] = [] + observed = 0 + while observed <= before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail("wheel_path_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("wheel_path_invalid") + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail("wheel_path_changed") + return payload + + +def _read_exact_sdist(path: Path) -> bytes: + if os.name != "posix" or path.name != _SDIST_NAME: + _fail("sdist_path_invalid") + required = ("O_CLOEXEC", "O_NOCTTY", "O_NOFOLLOW", "O_NONBLOCK") + if any(not hasattr(os, name) for name in required): + _fail("platform_unsupported") + flags = os.O_RDONLY | os.O_CLOEXEC | os.O_NOCTTY | os.O_NOFOLLOW | os.O_NONBLOCK + descriptor = -1 + try: + descriptor = os.open(path, flags) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size <= 0 + or before.st_size > _MAX_COMPRESSED_SDIST_BYTES + ): + _fail("sdist_path_invalid") + chunks: list[bytes] = [] + observed = 0 + while observed <= before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size + 1 - observed), + ) + if not chunk: + break + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except OSError: + _fail("sdist_path_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("sdist_path_invalid") + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail("sdist_path_changed") + return payload + + +def _safe_sdist_member_name(member: tarfile.TarInfo) -> str: + raw_name = member.name + if ( + not isinstance(raw_name, str) + or not raw_name + or "\\" in raw_name + or "\x00" in raw_name + or any(ord(character) < 32 or ord(character) == 127 for character in raw_name) + ): + _fail("sdist_member_unsafe") + normalized = ( + raw_name[:-1] if member.isdir() and raw_name.endswith("/") else raw_name + ) + path = PurePosixPath(normalized) + first = path.parts[0] if path.parts else "" + if ( + not normalized + or path.is_absolute() + or path.as_posix() != normalized + or any(part in ("", ".", "..") for part in path.parts) + or (len(first) >= 2 and first[0].isalpha() and first[1] == ":") + or (not member.isdir() and raw_name.endswith("/")) + ): + _fail("sdist_member_unsafe") + return normalized + + +def _inspect_sdist( + payload: bytes, + *, + source_date_epoch: int, +) -> tuple[InventoryMemberEvidence, ...]: + if ( + type(payload) is not bytes + or len(payload) < 18 + or payload[:4] != b"\x1f\x8b\x08\x00" + or int.from_bytes(payload[4:8], "little") != source_date_epoch + ): + _fail("sdist_not_canonical") + compressed = io.BytesIO(payload) + try: + with gzip.GzipFile(fileobj=compressed, mode="rb") as archive: + tar_payload = archive.read(_MAX_EXPANDED_SDIST_BYTES + 1) + except (EOFError, OSError, gzip.BadGzipFile, zlib.error): + _fail("sdist_archive_invalid") + if ( + not 0 < len(tar_payload) <= _MAX_EXPANDED_SDIST_BYTES + or len(tar_payload) % _TAR_BLOCK_BYTES + or len(tar_payload) < _TAR_TRAILER_BYTES + ): + _fail("sdist_archive_invalid") + + observed: list[InventoryMemberEvidence] = [] + observed_paths: set[str] = set() + total_file_bytes = 0 + archive_offset = -1 + try: + with tarfile.open(fileobj=io.BytesIO(tar_payload), mode="r:") as archive: + members = archive.getmembers() + archive_offset = archive.offset + if not 1 <= len(members) <= _MAX_SDIST_MEMBERS: + _fail("sdist_archive_invalid") + for member in members: + if ( + not (member.isdir() or member.isreg()) + or member.sparse is not None + or member.uid != 0 + or member.gid != 0 + or member.uname != "" + or member.gname != "" + or member.linkname + or member.pax_headers + or member.mtime != source_date_epoch + ): + _fail("sdist_member_unsafe") + name = _safe_sdist_member_name(member) + if name in observed_paths: + _fail("sdist_member_duplicate") + observed_paths.add(name) + if member.isdir(): + if member.size != 0 or member.mode != 0o755: + _fail("sdist_not_canonical") + observed.append( + InventoryMemberEvidence( + path=name, + kind="directory", + mode="0755", + size_bytes=0, + sha256=None, + ) + ) + continue + if ( + not 0 <= member.size <= _MAX_SDIST_MEMBER_BYTES + or member.mode not in (0o644, 0o755) + ): + _fail("sdist_member_unsafe") + total_file_bytes += member.size + if total_file_bytes > _MAX_SDIST_FILE_BYTES: + _fail("sdist_archive_invalid") + extracted = archive.extractfile(member) + if extracted is None: + _fail("sdist_archive_invalid") + member_payload = extracted.read(_MAX_SDIST_MEMBER_BYTES + 1) + if len(member_payload) != member.size: + _fail("sdist_archive_invalid") + observed.append( + InventoryMemberEvidence( + path=name, + kind="file", + mode=f"{member.mode:04o}", + size_bytes=len(member_payload), + sha256=hashlib.sha256(member_payload).hexdigest(), + ) + ) + except VerificationFailure: + raise + except (OSError, tarfile.TarError, UnicodeError, ValueError): + _fail("sdist_archive_invalid") + if ( + archive_offset < 0 + or len(tar_payload) - archive_offset < _TAR_TRAILER_BYTES + or any(tar_payload[archive_offset:]) + or [member.path for member in observed] + != sorted(member.path for member in observed) + ): + _fail("sdist_not_canonical") + return tuple(observed) + + +def _validate_member_name(name: str) -> None: + path = PurePosixPath(name) + if ( + not name + or "\\" in name + or path.is_absolute() + or any(part in ("", ".", "..") for part in path.parts) + ): + _fail("wheel_archive_invalid") + + +def _validate_zip_member(info: ZipInfo) -> None: + _validate_member_name(info.filename) + mode = info.external_attr >> 16 + if ( + info.is_dir() + or info.flag_bits & 0x1 + or info.file_size < 0 + or info.file_size > _MAX_MEMBER_BYTES + or not mode + or not stat.S_ISREG(mode) + ): + _fail("wheel_archive_invalid") + + +def _decode_record_digest(value: str) -> str: + if not value.startswith("sha256="): + _fail("record_invalid") + digest = value.removeprefix("sha256=") + try: + decoded = base64.urlsafe_b64decode(digest + "=" * (-len(digest) % 4)) + except (UnicodeError, ValueError): + _fail("record_invalid") + if len(decoded) != hashlib.sha256().digest_size: + _fail("record_invalid") + return digest + + +def _parse_record(payload: bytes) -> dict[str, RecordEntry]: + try: + text = payload.decode("utf-8") + rows = list(csv.reader(io.StringIO(text, newline=""))) + except (UnicodeError, csv.Error): + _fail("record_invalid") + records: dict[str, RecordEntry] = {} + for row in rows: + if len(row) != 3: + _fail("record_invalid") + path, raw_digest, raw_size = row + if not path or path in records: + _fail("record_invalid") + digest = _decode_record_digest(raw_digest) if raw_digest else None + if raw_size: + if not raw_size.isascii() or not raw_size.isdecimal(): + _fail("record_invalid") + size: int | None = int(raw_size) + else: + size = None + if (digest is None) != (size is None): + _fail("record_invalid") + records[path] = RecordEntry(path=path, digest=digest, size=size) + return records + + +def _record_digest(payload: bytes) -> str: + digest = hashlib.sha256(payload).digest() + return base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + + +def _assert_record_match(entry: RecordEntry, payload: bytes) -> None: + if entry.size != len(payload) or entry.digest != _record_digest(payload): + _fail("record_mismatch") + + +def _parse_metadata(payload: bytes) -> Mapping[str, str]: + message = BytesParser(policy=policy.default).parsebytes(payload) + if ( + message["Name"] != _DISTRIBUTION_NAME + or message["Version"] != _DISTRIBUTION_VERSION + or message["Requires-Python"] != ">=3.11" + or message.get_all("Requires-Dist", []) != [] + ): + _fail("wheel_metadata_invalid") + return dict(message.items()) + + +def _inspect_wheel(payload: bytes) -> WheelEvidence: + try: + with ZipFile(io.BytesIO(payload)) as archive: + infos = archive.infolist() + names = [info.filename for info in infos] + if len(names) != len(set(names)): + _fail("wheel_archive_invalid") + for info in infos: + _validate_zip_member(info) + if ( + frozenset(names) != _EXPECTED_WHEEL_MEMBERS + or sum(info.file_size for info in infos) > _MAX_EXPANDED_BYTES + ): + _fail("wheel_inventory_invalid") + members = {name: archive.read(name) for name in names} + member_info = {info.filename: info for info in infos} + except VerificationFailure: + raise + except (BadZipFile, KeyError, OSError, RuntimeError): + _fail("wheel_archive_invalid") + + _parse_metadata(members[f"{_DIST_INFO}/METADATA"]) + if members[f"{_DIST_INFO}/top_level.txt"] != b"netveil\nnetveil_bootstrap\n": + _fail("wheel_metadata_invalid") + wheel_metadata = BytesParser(policy=policy.default).parsebytes( + members[f"{_DIST_INFO}/WHEEL"] + ) + if ( + wheel_metadata["Wheel-Version"] != "1.0" + or wheel_metadata["Root-Is-Purelib"] != "true" + or wheel_metadata.get_all("Tag", []) != ["py3-none-any"] + ): + _fail("wheel_metadata_invalid") + + record_name = f"{_DIST_INFO}/RECORD" + records = _parse_record(members[record_name]) + if set(records) != set(members): + _fail("record_invalid") + for name, member_payload in members.items(): + entry = records[name] + if name == record_name: + if entry.digest is not None or entry.size is not None: + _fail("record_invalid") + else: + _assert_record_match(entry, member_payload) + + launcher = members[_WHEEL_SCRIPT] + launcher_mode = member_info[_WHEEL_SCRIPT].external_attr >> 16 + if not launcher_mode & stat.S_IXUSR: + _fail("launcher_invalid") + _validate_launcher_bytes(launcher) + return WheelEvidence( + payload=payload, + sha256=hashlib.sha256(payload).hexdigest(), + launcher=launcher, + members=tuple( + WheelMemberEvidence( + path=name, + sha256=hashlib.sha256(members[name]).hexdigest(), + size=len(members[name]), + mode=f"{stat.S_IMODE(member_info[name].external_attr >> 16):04o}", + ) + for name in sorted(members) + ), + ) + + +def _validate_launcher_bytes(payload: bytes) -> None: + if ( + not payload.startswith(b"#!/bin/sh\n") + or b'exec "$netveil_script_directory/python" -IESB "$0" "$@"\n' not in payload + or b"[project.scripts]" in payload + ): + _fail("launcher_invalid") + + +def _write_exclusive(path: Path, payload: bytes, *, mode: int) -> None: + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_CLOEXEC, + mode, + ) + offset = 0 + while offset < len(payload): + written = os.write(descriptor, payload[offset:]) + if written <= 0: + _fail("private_file_write_failed") + offset += written + os.fsync(descriptor) + except OSError: + _fail("private_file_write_failed") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("private_file_write_failed") + + +def _base_environment( + root: Path, layout: InstalledLayout | None = None +) -> dict[str, str]: + home = root / "home" + temporary = root / "tmp" + home.mkdir(exist_ok=True) + temporary.mkdir(exist_ok=True) + binary = layout.launcher.parent if layout is not None else root + return { + "HOME": str(home), + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": str(binary), + "PIP_CONFIG_FILE": os.devnull, + "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INDEX": "1", + "PIP_NO_INPUT": "1", + "PIP_ROOT_USER_ACTION": "ignore", + "TMPDIR": str(temporary), + } + + +def _kill_process_group(process: subprocess.Popen[bytes]) -> None: + try: + os.killpg(process.pid, signal.SIGKILL) + except (OSError, ProcessLookupError): + try: + process.kill() + except OSError: + pass + + +def _run_process( + argv: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str], +) -> ProcessResult: + try: + with ( + tempfile.TemporaryFile("w+b") as stdout_file, + tempfile.TemporaryFile("w+b") as stderr_file, + ): + process = subprocess.Popen( + list(argv), + cwd=cwd, + env=dict(env), + stdin=subprocess.DEVNULL, + stdout=stdout_file, + stderr=stderr_file, + close_fds=True, + start_new_session=True, + ) + try: + returncode = process.wait(timeout=_PROCESS_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + _kill_process_group(process) + process.wait() + _fail("process_timeout") + stdout_file.seek(0) + stderr_file.seek(0) + stdout = stdout_file.read(_MAX_PROCESS_OUTPUT_BYTES + 1) + stderr = stderr_file.read(_MAX_PROCESS_OUTPUT_BYTES + 1) + except OSError: + _fail("process_start_failed") + if ( + len(stdout) > _MAX_PROCESS_OUTPUT_BYTES + or len(stderr) > _MAX_PROCESS_OUTPUT_BYTES + ): + _fail("process_output_unbounded") + return ProcessResult(returncode=returncode, stdout=stdout, stderr=stderr) + + +def _create_fresh_install( + root: Path, + wheel: WheelEvidence, +) -> InstalledLayout: + pinned_wheel = root / _WHEEL_NAME + _write_exclusive(pinned_wheel, wheel.payload, mode=0o600) + prefix = root / "venv" + try: + venv.EnvBuilder( + system_site_packages=False, + clear=False, + symlinks=False, + with_pip=False, + ).create(prefix) + except (OSError, subprocess.SubprocessError): + _fail("venv_creation_failed") + python = prefix / "bin" / "python" + if not python.is_file(): + _fail("venv_creation_failed") + environment = _base_environment(root) + ensurepip = _run_process( + ( + str(python), + "-I", + "-m", + "ensurepip", + "--default-pip", + ), + cwd=root, + env=environment, + ) + if ensurepip.returncode != 0: + _fail("ensurepip_failed") + install = _run_process( + ( + str(python), + "-I", + "-m", + "pip", + "install", + "--no-index", + "--no-deps", + "--disable-pip-version-check", + "--no-cache-dir", + str(pinned_wheel), + ), + cwd=root, + env=environment, + ) + if install.returncode != 0: + _fail("wheel_install_failed") + return _inspect_install(prefix, python, wheel.launcher) + + +def _installed_record_path(site_root: Path, launcher: Path) -> str: + return PurePosixPath(os.path.relpath(launcher, site_root)).as_posix() + + +def _read_installed_regular( + path: Path, + *, + maximum: int, +) -> tuple[bytes, os.stat_result]: + descriptor = -1 + try: + descriptor = os.open( + path, + os.O_RDONLY | os.O_CLOEXEC | os.O_NOFOLLOW, + ) + before = os.fstat(descriptor) + if ( + not stat.S_ISREG(before.st_mode) + or before.st_size <= 0 + or before.st_size > maximum + ): + _fail("installed_layout_invalid") + chunks: list[bytes] = [] + observed = 0 + while observed < before.st_size: + chunk = os.read( + descriptor, + min(65_536, before.st_size - observed), + ) + if not chunk: + _fail("installed_layout_invalid") + chunks.append(chunk) + observed += len(chunk) + payload = b"".join(chunks) + after = os.fstat(descriptor) + except VerificationFailure: + raise + except OSError: + _fail("installed_layout_invalid") + finally: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + _fail("installed_layout_invalid") + if len(payload) != before.st_size or _identity(before) != _identity(after): + _fail("installed_layout_changed") + return payload, before + + +def _record_sha256_hex(digest: str | None) -> str | None: + if digest is None: + return None + try: + decoded = base64.urlsafe_b64decode(digest + "=" * (-len(digest) % 4)) + except (UnicodeError, ValueError): + _fail("record_invalid") + if len(decoded) != hashlib.sha256().digest_size: + _fail("record_invalid") + return decoded.hex() + + +def _inspect_install( + prefix: Path, + python: Path, + expected_launcher: bytes, +) -> InstalledLayout: + candidates = { + path.resolve() + for library in ("lib", "lib64") + for path in (prefix / library).glob(f"python*/site-packages/{_DIST_INFO}") + if path.is_dir() + } + if len(candidates) != 1: + _fail("installed_layout_invalid") + dist_info = candidates.pop() + site_root = dist_info.parent + launcher = prefix / "bin" / _LAUNCHER_NAME + bootstrap = site_root / _BOOTSTRAP_NAME + package_root = site_root / "netveil" + record_path = dist_info / "RECORD" + required = (launcher, bootstrap, package_root / "cli.py", record_path) + if any(not path.is_file() for path in required): + _fail("installed_layout_invalid") + launcher_payload, launcher_status = _read_installed_regular( + launcher, + maximum=_MAX_MEMBER_BYTES, + ) + record_payload, record_status = _read_installed_regular( + record_path, + maximum=_MAX_MEMBER_BYTES, + ) + if not launcher_status.st_mode & stat.S_IXUSR: + _fail("launcher_invalid") + if launcher_payload != expected_launcher: + _fail("launcher_identity_mismatch") + _validate_launcher_bytes(launcher_payload) + records = _parse_record(record_payload) + relative_launcher = _installed_record_path(site_root, launcher) + launcher_record = records.get(relative_launcher) + if launcher_record is None: + _fail("launcher_record_missing") + _assert_record_match(launcher_record, launcher_payload) + record_record_path = f"{_DIST_INFO}/RECORD" + record_record = records.get(record_record_path) + if ( + record_record is None + or record_record.digest is not None + or record_record.size is not None + ): + _fail("record_invalid") + if (dist_info / "entry_points.txt").exists(): + _fail("entry_points_present") + distribution = importlib.metadata.PathDistribution(dist_info) + if ( + list(distribution.entry_points) + or distribution.metadata["Name"] != _DISTRIBUTION_NAME + or distribution.version != _DISTRIBUTION_VERSION + ): + _fail("entry_points_present") + selected_paths = { + relative_launcher, + _BOOTSTRAP_NAME, + "netveil/__init__.py", + "netveil/cli.py", + "netveil/model.py", + "netveil/parser.py", + "netveil/privacy.py", + "netveil/py.typed", + f"{_DIST_INFO}/METADATA", + record_record_path, + f"{_DIST_INFO}/WHEEL", + f"{_DIST_INFO}/licenses/LICENSE", + f"{_DIST_INFO}/top_level.txt", + } + selected_rows = tuple( + InstalledRecordRowEvidence( + path=path, + sha256=_record_sha256_hex(records[path].digest), + size_bytes=records[path].size, + ) + for path in sorted(selected_paths & records.keys()) + ) + if not { + relative_launcher, + record_record_path, + }.issubset(row.path for row in selected_rows): + _fail("record_invalid") + evidence = InstalledEvidence( + launcher=InstalledFileEvidence( + logical_path="bin/netveil-audit", + mode=f"{stat.S_IMODE(launcher_status.st_mode):04o}", + sha256=hashlib.sha256(launcher_payload).hexdigest(), + size_bytes=len(launcher_payload), + ), + record=InstalledFileEvidence( + logical_path=f"site-packages/{record_record_path}", + mode=f"{stat.S_IMODE(record_status.st_mode):04o}", + sha256=hashlib.sha256(record_payload).hexdigest(), + size_bytes=len(record_payload), + ), + selected_record_rows=selected_rows, + ) + return InstalledLayout( + prefix=prefix, + python=python, + launcher=launcher, + site_root=site_root, + dist_info=dist_info, + bootstrap=bootstrap, + package_root=package_root, + record=record_path, + evidence=evidence, + ) + + +def _assert_no_private_output( + result: ProcessResult, + forbidden: Sequence[bytes], +) -> None: + output = result.stdout + result.stderr + if any(token and token in output for token in forbidden): + _fail("private_output_detected") + + +def _invoke( + layout: InstalledLayout, + arguments: Sequence[str], + *, + cwd: Path, + env: Mapping[str, str], + through_path: bool = False, +) -> ProcessResult: + command = _LAUNCHER_NAME if through_path else str(layout.launcher) + return _run_process((command, *arguments), cwd=cwd, env=env) + + +def _verify_user_commands( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> None: + forbidden = (str(root).encode(),) + expected_version = b"netveil-audit 0.3.0\n" + for arguments, exact_stdout in ( + (("--version",), expected_version), + (("--help",), None), + ): + direct = _invoke(layout, arguments, cwd=root, env=env) + path = _invoke( + layout, + arguments, + cwd=root, + env=env, + through_path=True, + ) + if ( + direct.returncode != 0 + or path.returncode != 0 + or direct.stderr + or path.stderr + or direct.stdout != path.stdout + or (exact_stdout is not None and direct.stdout != exact_stdout) + ): + _fail("command_contract_failed") + if exact_stdout is None and ( + not direct.stdout.startswith(b"usage: netveil-audit ") + or b"receipt" not in direct.stdout + or b"--version" not in direct.stdout + ): + _fail("help_contract_failed") + _assert_no_private_output(direct, forbidden) + _assert_no_private_output(path, forbidden) + + +def _updated_record( + payload: bytes, + *, + path: str, + replacement: bytes, +) -> bytes: + records = _parse_record(payload) + if path not in records: + _fail("record_rewrite_failed") + try: + rows = list(csv.reader(io.StringIO(payload.decode("utf-8"), newline=""))) + except (UnicodeError, csv.Error): + _fail("record_rewrite_failed") + output = io.StringIO(newline="") + writer = csv.writer(output, lineterminator="\n") + replaced = 0 + for row in rows: + if row[0] == path: + writer.writerow( + ( + path, + f"sha256={_record_digest(replacement)}", + str(len(replacement)), + ) + ) + replaced += 1 + else: + writer.writerow(row) + if replaced != 1: + _fail("record_rewrite_failed") + return output.getvalue().encode("utf-8") + + +@contextmanager +def _temporary_bytes(path: Path, payload: bytes) -> Iterator[None]: + existed = path.exists() + original = b"" + original_mode = 0 + if existed: + try: + status = path.lstat() + if not stat.S_ISREG(status.st_mode): + _fail("mutation_target_invalid") + original = path.read_bytes() + original_mode = stat.S_IMODE(status.st_mode) + except OSError: + _fail("mutation_target_invalid") + try: + path.write_bytes(payload) + yield + except OSError: + _fail("mutation_failed") + finally: + try: + if existed: + path.write_bytes(original) + path.chmod(original_mode) + else: + path.unlink(missing_ok=True) + except OSError: + _fail("mutation_restore_failed") + + +def _poisoned_environment( + env: Mapping[str, str], + *, + attack_root: Path, + marker: Path, +) -> dict[str, str]: + poisoned = dict(env) + poisoned.update( + { + "NETVEIL_VERIFY_MARKER": str(marker), + "PYTHONHOME": str(attack_root / "invalid-python-home"), + "PYTHONINSPECT": "1", + "PYTHONPATH": str(attack_root), + "PYTHONSTARTUP": str(attack_root / "startup.py"), + } + ) + return poisoned + + +def _verify_isolation( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> InterpreterEvidence: + attack_root = root / "startup-attack" + attack_root.mkdir() + marker = root / "startup-marker" + poisoned = _poisoned_environment(env, attack_root=attack_root, marker=marker) + attack_paths = ( + attack_root / "sitecustomize.py", + attack_root / "usercustomize.py", + attack_root / "startup.py", + layout.site_root / "sitecustomize.py", + layout.site_root / "usercustomize.py", + ) + try: + for path in attack_paths: + if path.exists(): + _fail("startup_probe_collision") + path.write_bytes(_ATTACK_SOURCE) + for through_path in (False, True): + result = _invoke( + layout, + ("--version",), + cwd=root, + env=poisoned, + through_path=through_path, + ) + if ( + result.returncode != 0 + or result.stdout != b"netveil-audit 0.3.0\n" + or result.stderr + or marker.exists() + ): + _fail("environment_isolation_failed") + _assert_no_private_output( + result, + (str(root).encode(), str(marker).encode()), + ) + + original_record = layout.record.read_bytes() + rewritten = _updated_record( + original_record, + path=_BOOTSTRAP_NAME, + replacement=_PROBE_SOURCE, + ) + with ( + _temporary_bytes(layout.bootstrap, _PROBE_SOURCE), + _temporary_bytes(layout.record, rewritten), + ): + probe = _invoke( + layout, + ("isolation-probe",), + cwd=root, + env=poisoned, + ) + try: + probe_document = json.loads(probe.stdout) + except (UnicodeError, json.JSONDecodeError): + _fail("isolation_probe_failed") + if not isinstance(probe_document, dict): + _fail("isolation_probe_failed") + implementation = probe_document.pop("implementation", None) + version = probe_document.pop("version", None) + cache_tag = probe_document.pop("cache_tag", None) + expected_version = ( + f"{sys.version_info.major}." + f"{sys.version_info.minor}." + f"{sys.version_info.micro}" + ) + if ( + probe.returncode != 0 + or probe.stderr + or probe_document != _EXPECTED_PROBE + or implementation != sys.implementation.name + or version != expected_version + or cache_tag != sys.implementation.cache_tag + or marker.exists() + ): + _fail("isolation_probe_failed") + _assert_no_private_output( + probe, + (str(root).encode(), str(marker).encode()), + ) + if ( + not isinstance(implementation, str) + or not isinstance(version, str) + or not isinstance(cache_tag, str) + ): + _fail("isolation_probe_failed") + evidence = InterpreterEvidence( + implementation=implementation, + version=version, + cache_tag=cache_tag, + ) + finally: + for path in attack_paths: + try: + path.unlink(missing_ok=True) + except OSError: + _fail("startup_probe_cleanup_failed") + return evidence + + +def _unchecked_hash_pyc(source: bytes, claimed_source: bytes) -> bytes: + code = compile(source, "", "exec", dont_inherit=True) + if not isinstance(code, CodeType): + _fail("pyc_probe_failed") + source_hash = importlib.util.source_hash(claimed_source) + return ( + importlib.util.MAGIC_NUMBER + + struct.pack(" None: + if marker.exists(): + _fail("adversarial_code_executed") + + +def _expect_successful_version( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], + marker: Path, +) -> None: + result = _invoke(layout, ("--version",), cwd=root, env=env) + if ( + result.returncode != 0 + or result.stdout != b"netveil-audit 0.3.0\n" + or result.stderr + ): + _fail("bytecode_inertness_failed") + _marker_absent(marker) + _assert_no_private_output( + result, + (str(root).encode(), str(marker).encode()), + ) + + +def _expect_artifact_failure( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], + marker: Path, +) -> None: + marker.unlink(missing_ok=True) + result = _invoke(layout, ("--version",), cwd=root, env=env) + if result.returncode != 10 or result.stdout or result.stderr != _ARTIFACT_FAILURE: + _fail("tamper_not_rejected") + _marker_absent(marker) + _assert_no_private_output( + result, + (str(root).encode(), str(marker).encode()), + ) + + +def _verify_bytecode_and_tamper( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> None: + marker = root / "adversarial-marker" + adversarial_env = dict(env) + adversarial_env["NETVEIL_VERIFY_MARKER"] = str(marker) + cache_tag = sys.implementation.cache_tag + if not cache_tag: + _fail("pyc_probe_failed") + + top_cache = layout.site_root / "__pycache__" + package_cache = layout.package_root / "__pycache__" + top_cache.mkdir(exist_ok=True) + package_cache.mkdir(exist_ok=True) + bootstrap_source = layout.bootstrap.read_bytes() + cli_path = layout.package_root / "cli.py" + cli_source = cli_path.read_bytes() + malicious_bootstrap_pyc = _unchecked_hash_pyc( + _ATTACK_SOURCE, + bootstrap_source, + ) + malicious_cli_pyc = _unchecked_hash_pyc(_ATTACK_SOURCE, cli_source) + + top_pyc = top_cache / f"netveil_bootstrap.{cache_tag}.pyc" + with _temporary_bytes(top_pyc, malicious_bootstrap_pyc): + _expect_successful_version( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + cli_pyc = package_cache / f"cli.{cache_tag}.pyc" + with _temporary_bytes(cli_pyc, malicious_cli_pyc): + _expect_successful_version( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + unknown_pyc = package_cache / f"unknown.{cache_tag}.pyc" + with _temporary_bytes(unknown_pyc, malicious_cli_pyc): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + launcher_payload = layout.launcher.read_bytes() + function_header = ( + b"def _execute_bootstrap(payload: bytes, bootstrap_path: Path) -> int:\n" + ) + if launcher_payload.count(function_header) != 1: + _fail("tamper_probe_unavailable") + launcher_tamper = launcher_payload.replace( + function_header, + function_header + + b' Path(os.environ["NETVEIL_VERIFY_MARKER"]).write_bytes(b"executed")\n', + ) + tamper_cases = ( + (layout.launcher, launcher_tamper), + (layout.bootstrap, bootstrap_source + b"\n" + _ATTACK_SOURCE), + (cli_path, cli_source + b"\n" + _ATTACK_SOURCE), + ) + for path, payload in tamper_cases: + with _temporary_bytes(path, payload): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + metadata_paths = ( + layout.dist_info / "METADATA", + layout.dist_info / "WHEEL", + layout.dist_info / "top_level.txt", + ) + for path in metadata_paths: + original = path.read_bytes() + with _temporary_bytes(path, original + b"\n# uncoordinated drift\n"): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + record_payload = layout.record.read_bytes() + record_drift = _updated_record( + record_payload, + path=_BOOTSTRAP_NAME, + replacement=b"uncoordinated bootstrap claim", + ) + with _temporary_bytes(layout.record, record_drift): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + unknown_source = layout.package_root / "unknown.py" + if unknown_source.exists(): + _fail("unknown_file_probe_collision") + with _temporary_bytes(unknown_source, _ATTACK_SOURCE): + _expect_artifact_failure( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + _expect_successful_version( + layout, + root=root, + env=adversarial_env, + marker=marker, + ) + + +def _canonical_json(document: object) -> bytes: + return json.dumps( + document, + allow_nan=False, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + + +def _verify_receipt_document( + payload: bytes, + *, + key: bytes, +) -> dict[str, object]: + if not payload.endswith(b"\n") or payload.endswith(b"\n\n"): + _fail("receipt_not_canonical") + try: + document = json.loads(payload) + except (UnicodeError, json.JSONDecodeError): + _fail("receipt_invalid") + if ( + not isinstance(document, dict) + or document.get("schema") != "netveil.aggregate-receipt.v1" + or _canonical_json(document) + b"\n" != payload + ): + _fail("receipt_not_canonical") + report = document.get("report") + digest = document.get("report_digest") + if not isinstance(report, dict) or not isinstance(digest, dict): + _fail("receipt_invalid") + counts = report.get("counts") + by_version = report.get("endpoint_occurrences_by_ip_version") + by_scope = report.get("endpoint_occurrences_by_scope") + duplicates = report.get("duplicates") + if ( + counts + != { + "endpoint_occurrences": 5, + "physical_lines": 6, + "source_bytes": len(_CORPUS), + "unique_endpoints": 4, + } + or not isinstance(by_version, dict) + or by_version.get("ipv4") != 4 + or by_version.get("ipv6") != 1 + or not isinstance(by_scope, dict) + or by_scope.get("documentation") != 5 + or not isinstance(duplicates, dict) + or duplicates.get("group_count") != 1 + ): + _fail("receipt_semantics_invalid") + expected_digest = hashlib.sha256(_canonical_json(report)).hexdigest() + if digest != {"algorithm": "sha256", "value": expected_digest}: + _fail("receipt_digest_invalid") + source_id = report.get("source_content_id") + if ( + not isinstance(source_id, str) + or not source_id.startswith("nvs1_") + or len(source_id) != 69 + ): + _fail("receipt_identifier_invalid") + + forbidden = ( + *_RAW_ENDPOINT_TOKENS, + hashlib.sha256(_CORPUS).hexdigest().encode(), + key, + key.hex().encode(), + base64.b64encode(key), + base64.urlsafe_b64encode(key), + ) + if any(token and token in payload for token in forbidden): + _fail("receipt_private_data_detected") + return dict(document) + + +def _verify_receipt( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> ReceiptEvidence: + corpus_path = root / "documentation-corpus.txt" + key_path = root / "private-receipt.key" + _write_exclusive(corpus_path, _CORPUS, mode=0o600) + key = os.urandom(32) + _write_exclusive(key_path, key, mode=0o600) + key_status = key_path.lstat() + if ( + not stat.S_ISREG(key_status.st_mode) + or stat.S_IMODE(key_status.st_mode) != 0o600 + or key_status.st_uid != os.geteuid() + or key_status.st_nlink != 1 + ): + _fail("private_key_policy_failed") + arguments = ("receipt", str(corpus_path), "--key-file", str(key_path)) + first = _invoke(layout, arguments, cwd=root, env=env) + second = _invoke(layout, arguments, cwd=root, env=env) + through_path = _invoke( + layout, + arguments, + cwd=root, + env=env, + through_path=True, + ) + if ( + first.returncode != 0 + or second.returncode != 0 + or through_path.returncode != 0 + or first.stderr + or second.stderr + or through_path.stderr + or first.stdout != second.stdout + or first.stdout != through_path.stdout + ): + _fail("receipt_command_failed") + _assert_no_private_output( + first, + (str(root).encode(), str(corpus_path).encode(), str(key_path).encode()), + ) + _verify_receipt_document(first.stdout, key=key) + return ReceiptEvidence( + corpus=corpus_path, + key=key_path, + output=first.stdout, + ) + + +def _capture_public_demo( + layout: InstalledLayout, + *, + root: Path, + env: Mapping[str, str], +) -> PublicDemoEvidence: + demo_root = root / "public-demo" + try: + demo_root.mkdir(mode=0o700) + except OSError: + _fail("public_demo_failed") + corpus_path = demo_root / "documentation-corpus.txt" + key_path = demo_root / "public-demo.key" + _write_exclusive(corpus_path, _CORPUS, mode=0o600) + _write_exclusive(key_path, _PUBLIC_DEMO_KEY, mode=0o600) + try: + key_status = key_path.lstat() + except OSError: + _fail("public_demo_failed") + if ( + not stat.S_ISREG(key_status.st_mode) + or stat.S_IMODE(key_status.st_mode) != 0o600 + or key_status.st_uid != os.geteuid() + or key_status.st_nlink != 1 + ): + _fail("public_demo_failed") + + version = _invoke( + layout, + ("--version",), + cwd=demo_root, + env=env, + through_path=True, + ) + receipt = _invoke( + layout, + ("receipt", corpus_path.name, "--key-file", key_path.name), + cwd=demo_root, + env=env, + through_path=True, + ) + if ( + version.returncode != 0 + or version.stdout != b"netveil-audit 0.3.0\n" + or version.stderr + or receipt.returncode != 0 + or receipt.stderr + ): + _fail("public_demo_failed") + _assert_no_private_output( + version, + (str(root).encode(), str(corpus_path).encode(), str(key_path).encode()), + ) + _assert_no_private_output( + receipt, + (str(root).encode(), str(corpus_path).encode(), str(key_path).encode()), + ) + document = _verify_receipt_document(receipt.stdout, key=_PUBLIC_DEMO_KEY) + try: + version_stdout = version.stdout.decode("ascii") + except UnicodeError: + _fail("public_demo_failed") + return PublicDemoEvidence( + corpus_sha256=hashlib.sha256(_CORPUS).hexdigest(), + corpus_size_bytes=len(_CORPUS), + corpus_physical_lines=_CORPUS.count(b"\n"), + public_key_sha256=hashlib.sha256(_PUBLIC_DEMO_KEY).hexdigest(), + public_key_size_bytes=len(_PUBLIC_DEMO_KEY), + version_stdout=version_stdout, + receipt=document, + receipt_stdout_sha256=hashlib.sha256(receipt.stdout).hexdigest(), + ) + + +def _strace_binary() -> Path: + if not sys.platform.startswith("linux"): + _fail("syscall_trace_unsupported") + located = shutil.which( + "strace", + path="/usr/bin:/bin:/usr/sbin:/sbin", + ) + if located is None: + _fail("strace_unavailable") + path = Path(located) + try: + status = path.lstat() + except OSError: + _fail("strace_unavailable") + if ( + not stat.S_ISREG(status.st_mode) + or not status.st_mode & stat.S_IXUSR + or status.st_mode & (stat.S_ISUID | stat.S_ISGID) + ): + _fail("strace_unavailable") + return path + + +def _read_trace_files(prefix: Path) -> tuple[bytes, ...]: + paths = sorted(prefix.parent.glob(f"{prefix.name}.*")) + if not paths: + _fail("syscall_trace_missing") + payloads: list[bytes] = [] + for path in paths: + try: + status = path.lstat() + if ( + not stat.S_ISREG(status.st_mode) + or status.st_size < 1 + or status.st_size > _MAX_PROCESS_OUTPUT_BYTES + ): + _fail("syscall_trace_invalid") + payload = path.read_bytes() + except OSError: + _fail("syscall_trace_invalid") + if len(payload) != status.st_size: + _fail("syscall_trace_invalid") + payloads.append(payload) + return tuple(payloads) + + +def _validate_syscall_trace( + payloads: Sequence[bytes], + *, + layout: InstalledLayout, + label: str = "unit", +) -> TraceEvidence: + if len(payloads) != 1: + _fail("post_launch_process_detected") + try: + lines = payloads[0].decode("utf-8").splitlines() + except UnicodeError: + _fail("syscall_trace_invalid") + if not lines: + _fail("syscall_trace_invalid") + exec_paths: list[str] = [] + exit_syscall_count = 0 + for line in lines: + syscall = line.split("(", 1)[0].strip() + if syscall == "execve": + match = re.match(r'^execve\("([^"\\]+)"', line) + if match is None or not line.rstrip().endswith("= 0"): + _fail("exec_chain_invalid") + exec_paths.append(match.group(1)) + elif syscall in ("exit", "exit_group"): + exit_syscall_count += 1 + else: + # The trace selector contains only process and network syscalls. + # Anything else is therefore a fork/clone/wait or network action. + _fail("network_or_process_activity_detected") + expected = [str(layout.launcher), str(layout.python)] + if exec_paths != expected or exit_syscall_count < 1: + _fail("exec_chain_invalid") + normalized = { + "exec_chain": ["installed_launcher", "installed_python"], + "exec_count": len(exec_paths), + "exit_syscall_count": exit_syscall_count, + "label": label, + "network_syscall_count": 0, + "post_launch_process_count": 0, + "process_count": len(payloads), + } + return TraceEvidence( + label=label, + normalized_sha256=hashlib.sha256(_canonical_json(normalized)).hexdigest(), + process_count=len(payloads), + exec_chain=("installed_launcher", "installed_python"), + exec_count=len(exec_paths), + exit_syscall_count=exit_syscall_count, + network_syscall_count=0, + post_launch_process_count=0, + ) + + +def _trace_invocation( + layout: InstalledLayout, + arguments: Sequence[str], + *, + root: Path, + env: Mapping[str, str], + label: str, + expected_stdout: bytes, +) -> TraceEvidence: + prefix = root / f"syscall-trace-{label}" + result = _run_process( + ( + str(_strace_binary()), + "-ff", + "-qq", + "-s", + "256", + "-e", + "trace=network,process", + "-o", + str(prefix), + "--", + str(layout.launcher), + *arguments, + ), + cwd=root, + env=env, + ) + if result.returncode != 0 or result.stdout != expected_stdout or result.stderr: + _fail("traced_command_failed") + _assert_no_private_output(result, (str(root).encode(),)) + return _validate_syscall_trace( + _read_trace_files(prefix), + layout=layout, + label=label, + ) + + +def _verify_syscall_traces( + layout: InstalledLayout, + receipt: ReceiptEvidence, + *, + root: Path, + env: Mapping[str, str], +) -> tuple[TraceEvidence, ...]: + version = _trace_invocation( + layout, + ("--version",), + root=root, + env=env, + label="version", + expected_stdout=b"netveil-audit 0.3.0\n", + ) + receipt_trace = _trace_invocation( + layout, + ( + "receipt", + str(receipt.corpus), + "--key-file", + str(receipt.key), + ), + root=root, + env=env, + label="receipt", + expected_stdout=receipt.output, + ) + return (receipt_trace, version) + + +def _platform_evidence() -> PlatformEvidence: + try: + uname = os.uname() + except (AttributeError, OSError): + _fail("platform_unsupported") + values = (sys.platform, uname.sysname, uname.release, uname.machine) + if any( + not isinstance(value, str) + or not value + or len(value) > 256 + or any(ord(character) < 32 or ord(character) == 127 for character in value) + for value in values + ): + _fail("platform_unsupported") + return PlatformEvidence( + sys_platform=sys.platform, + system=uname.sysname, + release=uname.release, + machine=uname.machine, + ) + + +def _validate_source_commit(source_commit: str) -> None: + if len(source_commit) != 40 or any( + character not in _LOWER_HEX for character in source_commit + ): + _fail("source_commit_invalid") + + +def verify_wheel( + path: Path, + *, + inventory_path: Path, + sdist_path: Path, + source_commit: str, +) -> VerificationSummary: + """Run the complete offline fresh-wheel verification.""" + + _validate_source_commit(source_commit) + inventory = _parse_release_inventory(_read_exact_inventory(inventory_path)) + if inventory.source_commit != source_commit: + _fail("inventory_source_commit_mismatch") + wheel_payload = _read_exact_wheel(path) + sdist_payload = _read_exact_sdist(sdist_path) + _bind_release_inventory( + inventory, + expected_source_commit=source_commit, + wheel_path=path, + wheel_payload=wheel_payload, + sdist_path=sdist_path, + sdist_payload=sdist_payload, + ) + sdist_members = _inspect_sdist( + sdist_payload, + source_date_epoch=inventory.source_date_epoch, + ) + _bind_sdist_members(inventory, sdist_members) + wheel = _inspect_wheel(wheel_payload) + with tempfile.TemporaryDirectory(prefix="netveil-wheel-verifier-") as raw_root: + root = Path(raw_root) + layout = _create_fresh_install(root, wheel) + environment = _base_environment(root, layout) + _verify_user_commands(layout, root=root, env=environment) + interpreter = _verify_isolation(layout, root=root, env=environment) + _verify_bytecode_and_tamper(layout, root=root, env=environment) + receipt = _verify_receipt(layout, root=root, env=environment) + public_demo = _capture_public_demo( + layout, + root=root, + env=environment, + ) + syscall_traces = _verify_syscall_traces( + layout, + receipt, + root=root, + env=environment, + ) + final_layout = _inspect_install( + layout.prefix, + layout.python, + wheel.launcher, + ) + installed = final_layout.evidence + if installed is None: + _fail("installed_layout_invalid") + return VerificationSummary( + source_commit=source_commit, + release_inventory=inventory, + installed=installed, + interpreter=interpreter, + platform=_platform_evidence(), + syscall_traces=syscall_traces, + public_demo=public_demo, + wheel_sha256=wheel.sha256, + wheel_size_bytes=len(wheel.payload), + wheel_members=wheel.members, + ) + + +def _write_text(stream: object, payload: str) -> bool: + writer = getattr(stream, "write", None) + flusher = getattr(stream, "flush", None) + if not callable(writer) or not callable(flusher): + return False + offset = 0 + try: + while offset < len(payload): + written = writer(payload[offset:]) + if ( + type(written) is not int + or written <= 0 + or written > len(payload) - offset + ): + return False + offset += written + flusher() + except OSError: + return False + return True + + +def main(argv: Sequence[str] | None = None) -> int: + """CLI entry point with stable, path-free output.""" + + arguments = list(sys.argv[1:] if argv is None else argv) + if arguments == ["--help"]: + text = ( + "usage: verify_fresh_wheel.py --source-commit COMMIT " + "--inventory release-inventory.json " + "--sdist netveil_audit-0.3.0.tar.gz " + "netveil_audit-0.3.0-py3-none-any.whl\n" + ) + return 0 if _write_text(sys.stdout, text) else 70 + if ( + len(arguments) != 7 + or arguments[0] != "--source-commit" + or arguments[2] != "--inventory" + or arguments[3].startswith("-") + or arguments[4] != "--sdist" + or arguments[5].startswith("-") + or arguments[6].startswith("-") + ): + _write_text(sys.stderr, "netveil-wheel-verifier: usage_error\n") + return 2 + try: + source_commit = arguments[1] + _validate_source_commit(source_commit) + summary = verify_wheel( + Path(arguments[6]), + inventory_path=Path(arguments[3]), + sdist_path=Path(arguments[5]), + source_commit=source_commit, + ) + rendered = _canonical_json(summary.document()).decode("ascii") + "\n" + return 0 if _write_text(sys.stdout, rendered) else 70 + except VerificationFailure as failure: + _write_text( + sys.stderr, + f"netveil-wheel-verifier: {failure.code}\n", + ) + return 1 + except Exception: # noqa: BLE001 - the CLI boundary must redact all internals. + _write_text(sys.stderr, "netveil-wheel-verifier: internal_error\n") + return 70 + + +if __name__ == "__main__": + raise SystemExit(main())