From 3a9d70010c53b46d597324da609f554b9556dc29 Mon Sep 17 00:00:00 2001 From: Mike Date: Sun, 15 Nov 2026 13:15:04 -0800 Subject: [PATCH] Verify project-history package transport before import --- docs/installation.md | 3 +- docs/project-history-migration.md | 74 ++++++++++++++ scripts/project_package.py | 157 ++++++++++++++++++++++++++++++ tests/test_project_package.py | 125 ++++++++++++++++++++++++ 4 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 docs/project-history-migration.md create mode 100644 scripts/project_package.py create mode 100644 tests/test_project_package.py diff --git a/docs/installation.md b/docs/installation.md index 5cc11af..3f9f700 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -72,7 +72,8 @@ ownership. A completed environment is not a running-controller acceptance result - **Existing host migration:** use [verified snapshot restore and grant reconciliation](private-network-migration.md), then test authenticated readiness/inference and selected-source access. Project working copies/history, GitHub credentials and controller state are separate - acceptance checks. The Foundry project-export/import artifact is not yet a + acceptance checks. [Project package transport verification](project-history-migration.md) + is available before import. The Foundry project-export/import artifact is not yet a supported general-purpose Forge command; #64 retains that work. - **Desktop MCP client only:** after environment installation, follow [MCP registration](mcp.md), using the remote HTTPS controller and explicit diff --git a/docs/project-history-migration.md b/docs/project-history-migration.md new file mode 100644 index 0000000..4aa4d21 --- /dev/null +++ b/docs/project-history-migration.md @@ -0,0 +1,74 @@ +# Project history migration: verify before importing + +Project working copies/history and controller private state are separate payloads. +Use [controller migration](private-network-migration.md) for identities, task +history and handoff receipts. This document covers the first supported project +package step: transport verification. General export/import remains tracked in +[#64](https://github.com/kudama/forge/issues/64); the original Foundry scripts are +private task artifacts, not a supported general importer. + +## Verify an extracted package + +`scripts/project_package.py` reads an already extracted version-1 project-history +package with `manifest.json`, `checksums.json` and `checkouts/`. The original +Foundry export uses that format. It never extracts an archive, invokes Git, runs +packaged scripts, downloads objects, writes a report or changes runtime grants. + +Before transfer, obtain the SHA-256 of `checksums.json` on the trusted source: + +```sh +shasum -a 256 /absolute/path/to/package/checksums.json +``` + +Retain that digest through an authenticated channel or a private migration record. +Do not calculate the expected value from an untrusted destination copy. A hash +included beside its payload does not authenticate itself. If transferring an +archive, independently verify its trusted archive digest before extraction with +your reviewed transfer/extraction procedure; this command does not secure archive +extraction. Keep the original export immutable and private. + +On the destination, replace the package path and `TRUSTED_SOURCE_DIGEST` with the +actual source value: + +```sh +.venv/bin/python scripts/project_package.py /absolute/path/to/package \ + --checksums-sha256 TRUSTED_SOURCE_DIGEST +``` + +The package root must be an existing canonical owner-only directory. Verification +requires exact agreement between the regular-file/symlink inventory and checksum +manifest, canonical relative names, matching bytes/sizes/executable flags and +consistent working-file records. It rejects hard-linked/special files, duplicate +JSON keys and duplicate/escaping checkout IDs. It does not follow directory +symlinks; a packaged symlink is checked as a link, without opening its target. +An extra import report or other new file makes the immutable export fail the +inventory check. Keep reports outside the exported payload rather than deleting +unrecognized files to make verification pass. + +The summary prints counts and the trusted digest, without project contents or +credentials. `git_history_checked: false` and `imported: false` are explicit: +matching transport bytes do not establish complete Git objects, refs/reflogs, +index/stash/working-tree equivalence, safe Git configuration or target readiness. +Do not run Git or scripts from a package merely because this check passed. + +Use a quiescent, trusted private package. This verifier is not a filesystem lock +against concurrent writers, an importer, a Git configuration sanitizer or a +replacement for an authenticated transfer. Limits are 16 MiB per JSON manifest +and 100,000 file/link entries. Names are canonical POSIX paths; checkout IDs use +ASCII letters, digits, underscores, hyphens and dots. A package outside these +boundaries needs review rather than automatic conversion. + +## Validation and remaining work + +2026-11-15: synthetic tests cover read-only verification, unchanged bytes/modes, +legitimate dotfiles/Unicode paths/trailing-dot names, transport tampering, extra +and missing files, executable flags, hard links, directory symlinks, malformed +manifests and a changed checksum file rejected against its original trust anchor. +No live Foundry package or original private migration archive was modified. + +Still required for supported end-to-end migration: explicit source inventory and +ignored-file classification, export of independent linked-worktree histories, +full object/ref/index/reflog/stash verification, authenticated account inventory, +canonical project cloning, resumable conflict-safe import, missing-object/cloud +placeholder handling and separate runtime grant reconciliation. Preserve the +existing Foundry evidence without claiming this verifier implements those steps. diff --git a/scripts/project_package.py b/scripts/project_package.py new file mode 100644 index 0000000..47e8da3 --- /dev/null +++ b/scripts/project_package.py @@ -0,0 +1,157 @@ +"""Verify an extracted project-history package; never import or execute its contents.""" +import argparse +import hashlib +import json +import os +from pathlib import Path, PurePosixPath +import re +import stat + +JSON_LIMIT = 16 * 1024 * 1024 + + +def package_path(value): + if not isinstance(value, str) or not value or '\0' in value or '\\' in value: + raise ValueError('Invalid package path') + path = PurePosixPath(value) + if (path.is_absolute() or not path.parts or '..' in path.parts + or path.as_posix() != value): + raise ValueError('Package paths must be canonical and relative') + return path + + +def read_json(path): + def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError('Duplicate manifest key') + result[key] = value + return result + with path.open('rb') as stream: + data = stream.read(JSON_LIMIT + 1) + if len(data) > JSON_LIMIT: + raise ValueError('Package manifest exceeds size limit') + return json.loads(data, object_pairs_hook=unique) + + +def digest(path): + result = hashlib.sha256() + with path.open('rb') as stream: + for block in iter(lambda:stream.read(1024 * 1024), b''): + result.update(block) + return result.hexdigest() + + +def fingerprint(path): + info = path.lstat() + if stat.S_ISLNK(info.st_mode): + return {'kind':'symlink', 'target':os.readlink(path)} + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1: + raise ValueError('Package contains a special or hard-linked file') + return {'kind':'file', 'sha256':digest(path), 'size':info.st_size, + 'executable':bool(info.st_mode & 0o111)} + + +def package_entries(base): + pending = [base] + entries = {} + while pending: + directory = pending.pop() + with os.scandir(directory) as scan: + for entry in scan: + path = Path(entry.path) + relative = path.relative_to(base).as_posix() + package_path(relative) + if entry.is_dir(follow_symlinks=False): + pending.append(path) + else: + entries[relative] = path + if len(entries) > 100000: + raise ValueError('Too many package files') + return entries + + +def verify(base, checksums_sha256): + base = base.expanduser().absolute() + mode = base.lstat().st_mode + if (base.resolve() != base or not stat.S_ISDIR(mode) or mode & 0o077): + raise ValueError('Package must be a canonical owner-only directory') + if not isinstance(checksums_sha256, str) or not re.fullmatch('[0-9a-f]{64}', checksums_sha256): + raise ValueError('Supply the trusted SHA-256 of checksums.json') + checksum_file = base/'checksums.json' + info = checksum_file.lstat() + if not stat.S_ISREG(info.st_mode) or info.st_nlink != 1 or info.st_size > JSON_LIMIT: + raise ValueError('Invalid checksums file') + if digest(checksum_file) != checksums_sha256: + raise ValueError('Checksums file does not match the trusted digest') + checksums = read_json(checksum_file) + if not isinstance(checksums, dict) or not checksums or len(checksums) > 100000: + raise ValueError('Invalid package checksums') + for relative, expected in checksums.items(): + package_path(relative) + if not isinstance(expected, dict): + raise ValueError('Invalid file fingerprint') + if expected.get('kind') == 'file': + if (set(expected) != {'kind','sha256','size','executable'} + or not isinstance(expected['sha256'], str) + or not re.fullmatch('[0-9a-f]{64}', expected['sha256']) + or type(expected['size']) is not int or expected['size'] < 0 + or type(expected['executable']) is not bool): + raise ValueError('Invalid regular file fingerprint') + elif expected.get('kind') == 'symlink': + if set(expected) != {'kind','target'} or not isinstance(expected['target'], str): + raise ValueError('Invalid symlink fingerprint') + else: + raise ValueError('Unknown file fingerprint') + entries = package_entries(base) + if set(entries) != set(checksums) | {'checksums.json'} or 'checksums.json' in checksums: + raise ValueError('Package file inventory differs from checksums') + for relative, expected in checksums.items(): + if fingerprint(entries[relative]) != expected: + raise ValueError('Package fingerprint mismatch') + manifest_file = entries.get('manifest.json') + if manifest_file is None or checksums['manifest.json']['kind'] != 'file': + raise ValueError('Manifest must be a verified regular file') + manifest = read_json(manifest_file) + if (not isinstance(manifest, dict) or type(manifest.get('version')) is not int + or manifest['version'] != 1 or not isinstance(manifest.get('checkouts'), list)): + raise ValueError('Unsupported project package manifest') + seen = set() + for checkout in manifest['checkouts']: + if not isinstance(checkout, dict): + raise ValueError('Invalid checkout record') + name = checkout.get('id') + if (not isinstance(name, str) or not re.fullmatch('[A-Za-z0-9][A-Za-z0-9_.-]*', name) + or name in seen): + raise ValueError('Invalid or duplicate checkout ID') + seen.add(name) + if not isinstance(checkout.get('files'), dict): + raise ValueError('Invalid working-file manifest') + for relative, expected in checkout['files'].items(): + path = package_path(relative) + if '.git' in path.parts: + raise ValueError('Working files must not include Git metadata') + if checksums.get('checkouts/'+name+'/'+relative) != expected: + raise ValueError('Working-file manifest differs from transport inventory') + prefix = 'checkouts/'+name+'/.git/' + if not any(relative.startswith(prefix) for relative in checksums): + raise ValueError('Checkout has no preserved Git metadata') + # Recheck manifests after validation. This is not a lock against concurrent writers. + if digest(checksum_file) != checksums_sha256 or digest(manifest_file) != checksums['manifest.json']['sha256']: + raise ValueError('Package manifests changed during verification') + return {'verified':True, 'package_version':1, 'checkouts':len(seen), + 'files_checked':len(checksums), 'checksums_sha256':checksums_sha256, + 'git_history_checked':False, 'imported':False, 'runtime_grants_changed':False} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('package', type=Path) + parser.add_argument('--checksums-sha256', required=True) + args = parser.parse_args() + print(json.dumps(verify(args.package, args.checksums_sha256))) + + +if __name__ == '__main__': + main() diff --git a/tests/test_project_package.py b/tests/test_project_package.py new file mode 100644 index 0000000..dd5a4e9 --- /dev/null +++ b/tests/test_project_package.py @@ -0,0 +1,125 @@ +import json +from pathlib import Path, PurePosixPath + +import pytest + +from scripts import project_package as package + + +@pytest.mark.parametrize('value', ['', '.', '..', '../file', '/file', 'a/../b', + './a', 'a/./b', 'a//b', 'a/', 'a\\b', '\0', None, 3]) +def test_package_paths_reject_ambiguous_and_escaping_names(value): + with pytest.raises(ValueError):package.package_path(value) + + +@pytest.mark.parametrize('value', ['.git/config', 'folder/file.', '書籍/notes.md']) +def test_package_paths_preserve_legitimate_names(value): + assert package.package_path(value) == PurePosixPath(value) + + +@pytest.fixture +def exported(tmp_path): + base = tmp_path/'package';base.mkdir(mode=0o700) + checkout = base/'checkouts/01-example';checkout.mkdir(parents=True) + (checkout/'.git').mkdir() + (checkout/'.git/HEAD').write_text('ref: refs/heads/main\n') + (checkout/'file.txt').write_text('private working content') + (checkout/'link').symlink_to('/outside/not-read') + manifest = {'version':1, 'checkouts':[{'id':'01-example', 'files':{ + name:package.fingerprint(checkout/name) for name in ('file.txt','link')}}]} + (base/'manifest.json').write_text(json.dumps(manifest)) + return base + + +def seal(base): + checksums = {str(path.relative_to(base)):package.fingerprint(path) + for path in base.rglob('*') if (path.is_file() or path.is_symlink()) + and path.name != 'checksums.json'} + path = base/'checksums.json';path.write_text(json.dumps(checksums)) + return package.digest(path) + + +def snapshot(base): + return {str(path.relative_to(base)):(path.lstat().st_mode, + path.read_bytes() if path.is_file() and not path.is_symlink() else None) + for path in [base, *base.rglob('*')]} + + +def test_verification_is_read_only_and_reports_its_boundary(exported): + trusted = seal(exported);before = snapshot(exported) + result = package.verify(exported, trusted) + assert result == {'verified':True, 'package_version':1, 'checkouts':1, + 'files_checked':4, 'checksums_sha256':trusted, 'git_history_checked':False, + 'imported':False, 'runtime_grants_changed':False} + assert snapshot(exported) == before + assert 'private working content' not in json.dumps(result) + + +@pytest.mark.parametrize('change', ['extra','missing','content','mode','hardlink','directory_link']) +def test_transport_changes_are_rejected_before_import(exported, change): + trusted = seal(exported) + file = exported/'checkouts/01-example/file.txt' + if change == 'extra':(exported/'extra').write_text('not listed') + elif change == 'missing':file.unlink() + elif change == 'content':file.write_text('changed') + elif change == 'mode':file.chmod(0o700) + elif change == 'hardlink':(exported/'extra').hardlink_to(file) + else: + metadata = exported/'checkouts/01-example/.git' + moved = exported.parent/'outside';metadata.rename(moved) + metadata.symlink_to(moved, target_is_directory=True) + with pytest.raises(ValueError):package.verify(exported, trusted) + + +def test_symlink_fingerprint_never_opens_target(exported, monkeypatch): + trusted = seal(exported) + original = Path.open + def guarded(path, *args, **kwargs): + assert path.name != 'link' + return original(path, *args, **kwargs) + monkeypatch.setattr(Path, 'open', guarded) + assert package.verify(exported, trusted)['verified'] + + +@pytest.mark.parametrize('change', ['escape','duplicate_id','manifest_symlink','mismatch','bool_size']) +def test_invalid_package_structure_is_rejected(exported, change): + manifest_file = exported/'manifest.json' + manifest = json.loads(manifest_file.read_text()) + if change == 'escape':manifest['checkouts'][0]['id'] = '../outside' + elif change == 'duplicate_id':manifest['checkouts'] *= 2 + elif change == 'mismatch':manifest['checkouts'][0]['files']['file.txt']['size'] = 1 + elif change == 'manifest_symlink': + moved = exported.parent/'manifest.json';manifest_file.rename(moved) + manifest_file.symlink_to(moved) + if change != 'manifest_symlink':manifest_file.write_text(json.dumps(manifest)) + trusted = seal(exported) + if change == 'bool_size': + checksums = json.loads((exported/'checksums.json').read_text()) + checksums['checkouts/01-example/file.txt']['size'] = True + (exported/'checksums.json').write_text(json.dumps(checksums)) + trusted = package.digest(exported/'checksums.json') + with pytest.raises(ValueError):package.verify(exported, trusted) + + +def test_resealed_tampering_does_not_match_original_trust_anchor(exported): + trusted = seal(exported) + (exported/'checkouts/01-example/file.txt').write_text('tampered') + seal(exported) + with pytest.raises(ValueError, match='trusted digest'):package.verify(exported, trusted) + + +def test_duplicate_checksum_keys_are_rejected(exported): + seal(exported) + path = exported/'checksums.json';path.write_text('{"a":{},"a":{}}') + with pytest.raises(ValueError, match='Duplicate'):package.verify(exported, package.digest(path)) + + +@pytest.mark.parametrize('change', ['symlink','public','bad_digest']) +def test_unsafe_package_root_or_anchor_is_rejected(exported, change): + trusted = seal(exported) + if change == 'symlink': + link = exported.parent/'link';link.symlink_to(exported, target_is_directory=True) + exported = link + elif change == 'public':exported.chmod(0o755) + else:trusted = 'not-a-digest' + with pytest.raises(ValueError):package.verify(exported, trusted)