diff --git a/automation/run_reddit_baseline.ps1 b/automation/run_reddit_baseline.ps1 index ff33b41..968d194 100644 --- a/automation/run_reddit_baseline.ps1 +++ b/automation/run_reddit_baseline.ps1 @@ -99,12 +99,14 @@ function Assert-RedditSourcePackage { } function Invoke-SourcePublication { - param([Parameter(Mandatory = $true)]$Snapshot) + param([Parameter(Mandatory = $true)]$Snapshot, [string]$ExpectedReceipt) try { Assert-PreparedMainSha Run-Step "git checkout branch" @("git", "checkout", "-b", $branch) Run-Step "git add target files" (@("git", "add", "--") + $filesToStage) + $admittedIndex = Assert-LocalRedditAdmission -Index -ExpectedReceipt $ExpectedReceipt Run-Step "git commit" @("git", "commit", "-m", "chore(data): refresh reddit source package $utcDate") + Assert-LocalRedditAdmission -ExpectedIndex $admittedIndex -ExpectedReceipt $ExpectedReceipt | Out-Null Run-Step "git push" @("git", "push", "-u", "origin", $branch) $body = @" ## Summary @@ -139,6 +141,26 @@ function Invoke-SourcePublication { } } +function Assert-LocalRedditAdmission { + param([switch]$Index, [switch]$Worktree, [string]$ExpectedIndex, [string]$ExpectedReceipt) + Assert-RedditGitEnvironment + $arguments = @("-I", "-B", (Join-Path $repo "scripts/check_reddit_package_admission.py"), "--root", $repo, "--base", $PreparedMainSha) + if ($Worktree) { $arguments += "--worktree" } + elseif ($Index) { $arguments += @("--index", "--expected-receipt", $ExpectedReceipt) } + else { + $head = (git rev-parse HEAD).Trim() + if ($LASTEXITCODE -ne 0) { throw "Cannot resolve committed package SHA" } + $arguments += @("--head", $head, "--expected-index", $ExpectedIndex, "--expected-receipt", $ExpectedReceipt) + } + $result = @(& $py @arguments) + if ($LASTEXITCODE -ne 0 -or $result.Count -ne 1) { throw "Local Reddit package admission failed" } + $admission = $result[0] | ConvertFrom-Json + $identity = if ($Worktree) { $admission.receipt_fingerprint } else { $admission.index_fingerprint } + if ($admission.classification -ne "source-package" -or $admission.eligibility -ne "pass" -or + $identity -notmatch '^[0-9a-f]{64}$') { throw "Local admission did not accept a package" } + return [string]$identity +} + Set-Location $repo Assert-CleanWorktree Assert-PreparedMainSha @@ -162,6 +184,7 @@ $env:REDDIT_RSS_MAX_ATTEMPTS = "2" try { Run-Step "reddit source extraction" @($py, "backend\reddit_etl.py", "--source-package") Assert-RedditSourcePackage -BaselineMentions $baselineMentions + $sourceReceipt = Assert-LocalRedditAdmission -Worktree Restore-RedditOutputSnapshotPaths -Snapshot $outputSnapshot -RelativePaths $ignoredOutputs } catch { @@ -172,5 +195,5 @@ catch { throw $failure } -$prUrl = Invoke-SourcePublication -Snapshot $outputSnapshot +$prUrl = Invoke-SourcePublication -Snapshot $outputSnapshot -ExpectedReceipt $sourceReceipt Write-Output "PR_URL=$prUrl" diff --git a/scripts/check_reddit_package_admission.py b/scripts/check_reddit_package_admission.py index c01633e..8b8f160 100644 --- a/scripts/check_reddit_package_admission.py +++ b/scripts/check_reddit_package_admission.py @@ -1,8 +1,10 @@ """Admit committed Reddit packages as data using trusted code and a real UTC clock.""" +import argparse import csv from datetime import datetime, timezone import io +import hashlib import json import os from pathlib import Path @@ -14,6 +16,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "backend")) from reddit_source_package import producer_minimum_mentions, validate_reddit_source_package # noqa: E402 +from reddit_source_package import _unique_json_keys # noqa: E402 PATHS = ("datos/reddit_sentimiento_frameworks.csv", "datos/reddit_temas_emergentes.csv", "datos/source_packages/reddit/receipt.json") @@ -57,6 +60,11 @@ def admit(root, base, head, *, now=None): if len(entry) != 4 or entry[:2] != ["100644", "blob"] or entry[3] != path: raise ValueError("Package requires three regular 100644 files") blobs[path] = git(root, "cat-file", "blob", entry[2]) + return admit_blobs(root, base, blobs, now=now) + + +def admit_blobs(root, base, blobs, *, now=None): + """One provenance and coverage contract for committed and staged bytes.""" current = now if now is not None else datetime.now(timezone.utc) if current.tzinfo is None: raise ValueError("Admission clock must be UTC-aware") @@ -80,6 +88,95 @@ def admit(root, base, head, *, now=None): "posts_count_claim": package["posts_count"], "mentions_total": package["mentions_total"]} +def manifest(root, head=None): + """Capture every path, mode and object; never manufacture a tree from the worktree.""" + entries = {} + raw = git(root, "ls-tree", "-r", "-z", head) if head else git(root, "ls-files", "--stage", "-z") + for record in raw.split(b"\0")[:-1]: + metadata, path = record.split(b"\t", 1) + mode, middle, last = metadata.split() + oid, stage = (last, b"0") if head else (middle, last) + if stage != b"0" or path in entries: + raise ValueError("Unmerged or duplicate index entry") + entries[path] = (mode, oid) + return entries + + +def fingerprint(entries): + return hashlib.sha256(b"".join(path + b"\0" + mode + b" " + oid + b"\0" + for path, (mode, oid) in sorted(entries.items()))).hexdigest() + + +def receipt_identity(data): + receipt = json.loads(data.decode("utf-8"), object_pairs_hook=_unique_json_keys) + return hashlib.sha256(json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + + +def admit_worktree(root, base, *, now=None): + root = Path(root) + base = commit(root, base) + blobs = {path: (root / path).read_bytes() for path in PATHS} + result = admit_blobs(root, base, blobs, now=now) + if git(root, "rev-parse", "HEAD").decode().strip() != base or any((root / path).read_bytes() != data for path, data in blobs.items()): + raise ValueError("Source package or HEAD changed during worktree admission") + result["receipt_fingerprint"] = receipt_identity(blobs[PATHS[2]]) + return result + + +def admit_local(root, base, *, head=None, expected_index=None, expected_receipt=None, now=None): + """Require a package and preserve the exact admitted complete index across commit hooks.""" + root = Path(root) + base = commit(root, base) + before = manifest(root) + if head is not None: + head = commit(root, head) + if git(root, "rev-list", "--parents", "-n", "1", head).decode().split() != [head, base]: + raise ValueError("Committed package must have prepared main as its sole parent") + entries = manifest(root, head) + if not expected_index or fingerprint(entries) != expected_index or before != entries: + raise ValueError("Committed tree or index differs from admitted index") + else: + entries = before + baseline = manifest(root, base) + changed = {path for path in baseline.keys() | entries.keys() if baseline.get(path) != entries.get(path)} + if not changed or not changed <= {path.encode() for path in PATHS} or any(path not in entries for path in changed): + raise ValueError("Local publication requires only added or modified package paths") + blobs = {} + for path in PATHS: + mode, oid = entries.get(path.encode(), (None, None)) + if mode != b"100644": + raise ValueError("Package requires three regular 100644 files") + blobs[path] = git(root, "cat-file", "blob", oid.decode()) + result = admit_blobs(root, base, blobs, now=now) + if receipt_identity(blobs[PATHS[2]]) != expected_receipt: + raise ValueError("Package differs from initially validated source receipt") + if manifest(root) != before or git(root, "rev-parse", "HEAD").decode().strip() != (head or base): + raise ValueError("Index or HEAD changed during local admission") + result["index_fingerprint"] = fingerprint(entries) + return result + + +def local_main(argv): + parser = argparse.ArgumentParser(description="Read-only local Reddit package admission") + parser.add_argument("--root", type=Path, required=True) + parser.add_argument("--base", required=True) + view = parser.add_mutually_exclusive_group(required=True) + view.add_argument("--index", action="store_true") + view.add_argument("--worktree", action="store_true") + view.add_argument("--head") + parser.add_argument("--expected-index") + parser.add_argument("--expected-receipt") + args = parser.parse_args(argv) + try: + result = admit_worktree(args.root, args.base) if args.worktree else admit_local( + args.root, args.base, head=args.head, expected_index=args.expected_index, expected_receipt=args.expected_receipt) + print(json.dumps(result)) + except (ValueError, OSError, KeyError, UnicodeError, subprocess.TimeoutExpired) as exc: + print("Local admission failed: " + str(exc)) + return 1 + return 0 + + def event_binding(event, repository): pr = event["pull_request"] if repository != REPOSITORY or event["repository"]["full_name"] != repository or pr["base"]["ref"] != "main": @@ -130,4 +227,4 @@ def main(): if __name__ == "__main__": - sys.exit(main()) + sys.exit(local_main(sys.argv[1:]) if len(sys.argv) > 1 else main()) diff --git a/scripts/prepare_reddit_producer.py b/scripts/prepare_reddit_producer.py index 2776e1f..a165f61 100644 --- a/scripts/prepare_reddit_producer.py +++ b/scripts/prepare_reddit_producer.py @@ -11,7 +11,8 @@ ORIGIN = "https://github.com/Sam-24-dev/Technology-trend-analysis-platform" CONTROLS = ("automation/run_reddit_baseline_guarded.ps1", "automation/run_reddit_baseline.ps1", - "automation/reddit_output_transaction.psm1", "scripts/prepare_reddit_producer.py") + "automation/reddit_output_transaction.psm1", "scripts/prepare_reddit_producer.py", + "scripts/check_reddit_package_admission.py", "backend/reddit_source_package.py") GIT_SELECTORS = (r"^GIT_(DIR|WORK_TREE|COMMON_DIR|INDEX_FILE|OBJECT_DIRECTORY|ALTERNATE_OBJECT_DIRECTORIES|" r"NAMESPACE|.*PREFIX|SHALLOW_FILE|GRAFT_FILE|CONFIG.*|CEILING_DIRECTORIES|" r"DISCOVERY_ACROSS_FILESYSTEM|IMPLICIT_WORK_TREE|REPLACE_REF_BASE)$") diff --git a/tests/test_reddit_package_admission.py b/tests/test_reddit_package_admission.py index f9f3233..a89bb38 100644 --- a/tests/test_reddit_package_admission.py +++ b/tests/test_reddit_package_admission.py @@ -128,6 +128,130 @@ def test_code_only_skips_historical_package(candidate): assert gate.admit(repo, base, save(repo), now=NOW)["eligibility"] == "not-applicable" +@pytest.mark.parametrize("newline", ["\n", "\r\n"]) +def test_local_exact_index_and_commit_without_ci(candidate, monkeypatch, newline, capsys): + repo, base, receipt = candidate + (repo / gate.PATHS[2]).write_bytes((json.dumps(receipt) + newline).encode()) + original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes()) + gate.git(repo, "add", "--all") + staged = gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW) + original = gate.git + def local_git(root, *args): + assert args[0] not in {"fetch", "push", "remote"} + return original(root, *args) + monkeypatch.setattr(gate, "git", local_git) + monkeypatch.setattr(gate, "publish_check", lambda *args: pytest.fail("Local check publication")) + assert original(repo, "rev-parse", base + ":" + gate.PATHS[0]) == original(repo, "rev-parse", ":" + gate.PATHS[0]) + head = save(repo) + assert gate.admit_local(repo, base, head=head, expected_index=staged["index_fingerprint"], expected_receipt=original_receipt, now=NOW) == staged + finish = datetime.now(timezone.utc).replace(microsecond=0) + write_package(repo, finish=finish) + original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes()) + gate.git(repo, "add", "--all") + valid_index = gate.admit_local(repo, head, expected_receipt=original_receipt, now=finish) + head = save(repo) + expected = valid_index["index_fingerprint"] + assert gate.fingerprint(gate.manifest(repo, head)) == expected + assert gate.admit(repo, base, head, now=finish)["eligibility"] == "pass" + assert gate.receipt_identity(gate.git(repo, "cat-file", "blob", head + ":" + gate.PATHS[2])) == original_receipt + assert gate.local_main(["--root", str(repo), "--base", base, "--head", head, + "--expected-index", expected, "--expected-receipt", original_receipt]) == 1 + assert capsys.readouterr().out.strip() == "Local admission failed: Committed package must have prepared main as its sole parent" + + +@pytest.mark.parametrize("damage", ["empty", "code", "extra", "delete", "rename", "mode", "symlink", "unmerged", "tamper", "duplicate"]) +def test_local_bad_index_is_read_only(candidate, damage): + repo, base, _ = candidate + original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes()) + if damage in {"empty", "code"}: + gate.git(repo, "restore", "--worktree", "--", *gate.PATHS) + if damage in {"code", "extra"}: + (repo / "unrelated.py").write_text("pass\n") + if damage == "delete": + (repo / gate.PATHS[0]).unlink() + if damage == "rename": + (repo / gate.PATHS[0]).rename(repo / "renamed.csv") + if damage == "tamper": + (repo / gate.PATHS[1]).write_bytes(b"altered") + if damage == "duplicate": + path = repo / gate.PATHS[2] + path.write_text(path.read_text().replace('"source":', '"source":"duplicate","source":')) + gate.git(repo, "add", "--all") + oid = gate.git(repo, "rev-parse", base + ":" + gate.PATHS[0]).decode().strip() + if damage in {"mode", "symlink"}: + gate.git(repo, "update-index", "--cacheinfo", "100755" if damage == "mode" else "120000", oid, gate.PATHS[0]) + if damage == "unmerged": + gate.git(repo, "update-index", "--force-remove", "--", gate.PATHS[0]) + gate.subprocess.run(["git", "-C", str(repo), "update-index", "--index-info"], + input=f"100644 {oid} 1\t{gate.PATHS[0]}\n".encode(), check=True) + before = (gate.git(repo, "show-ref"), (repo / ".git/index").read_bytes()) + with pytest.raises(ValueError): + gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW) + assert before == (gate.git(repo, "show-ref"), (repo / ".git/index").read_bytes()) + + +@pytest.mark.parametrize("baseline,total,finish", [(300, 399, NOW), (300, 400, NOW), (501, 425, NOW), (501, 426, NOW), + (501, 500, NOW + timedelta(seconds=1)), (501, 500, NOW - timedelta(days=1)), + (501, 500, NOW - timedelta(days=9))]) +def test_local_utc_and_integer_ceiling(candidate, baseline, total, finish): + repo, _, _ = candidate + write_package(repo, baseline, NOW - timedelta(days=9)) + base = save(repo) + write_package(repo, total, finish) + original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes()) + gate.git(repo, "add", "--all") + if total in {400, 426}: + assert gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW)["minimum_mentions"] == total + else: + with pytest.raises(ValueError): + gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW) + + +@pytest.mark.parametrize("committed", [False, True]) +def test_local_valid_but_different_package_is_tampering(candidate, committed): + repo, base, _ = candidate + original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes()) + gate.git(repo, "add", "--all") + admitted = gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW)["index_fingerprint"] + head = save(repo) if not committed else None + write_package(repo, 801) + gate.git(repo, "add", "--all") + head = save(repo) if committed else head + with pytest.raises(ValueError, match="differs"): + gate.admit_local(repo, base, head=head, expected_index=admitted, expected_receipt=original_receipt, now=NOW) + + +def test_local_cli_real_clock_never_calls_ci(candidate, monkeypatch, capsys): + repo, base, _ = candidate + finish = datetime.now(timezone.utc).replace(microsecond=0) + write_package(repo, finish=finish) + gate.git(repo, "add", "--all") + monkeypatch.setattr(gate, "main", lambda: pytest.fail("CI entry")) + monkeypatch.setattr(gate, "urlopen", lambda *args, **kwargs: pytest.fail("Network")) + args = ["--root", str(repo), "--base", base] + assert gate.local_main([*args, "--worktree"]) == 0 + original_receipt = json.loads(capsys.readouterr().out)["receipt_fingerprint"] + assert gate.local_main([*args, "--index"]) == 1 + assert "initially validated source" in capsys.readouterr().out + args += ["--expected-receipt", original_receipt] + assert gate.local_main([*args, "--head", ""]) == 1 + assert "exact commit SHA" in capsys.readouterr().out + assert gate.local_main([*args, "--index"]) == 0 + staged = json.loads(capsys.readouterr().out) + head = save(repo) + assert gate.local_main([*args, "--head", head, "--expected-index", staged["index_fingerprint"]]) == 0 + assert json.loads(capsys.readouterr().out) == staged + + +def test_local_coherent_staged_replacement_rejects_original_source(candidate): + repo, base, _ = candidate + original = gate.admit_worktree(repo, base, now=NOW)["receipt_fingerprint"] + write_package(repo, 801) + gate.git(repo, "add", "--all") + with pytest.raises(ValueError, match="initially validated source"): + gate.admit_local(repo, base, expected_receipt=original, now=NOW) + + def test_floor_ceiling_and_verified_baseline(candidate): assert (301 * 85 + 99) // 100 == 256 and (300 * 85 + 99) // 100 == 255 assert producer_minimum_mentions(301) == producer_minimum_mentions(300) == 400 diff --git a/tests/test_reddit_producer_preparation.py b/tests/test_reddit_producer_preparation.py index 4b078dd..f26b838 100644 --- a/tests/test_reddit_producer_preparation.py +++ b/tests/test_reddit_producer_preparation.py @@ -276,6 +276,21 @@ def test_revision_assertions_precede_baseline_and_publication(): assert publication.index("Assert-PreparedMainSha") < publication.index('Run-Step "git checkout branch"') +@pytest.mark.parametrize("control", ["scripts/check_reddit_package_admission.py", "backend/reddit_source_package.py"]) +def test_preparation_rejects_changed_admission_controls(producer, control): + repo, origin, baseline, target = producer + assert control in prep.CONTROLS + git(repo, "reset", "--hard", target) + (repo / control).write_text("changed\n") + git(repo, "commit", "-qam", "test: change admission control") + git(repo, "push", "-q", "origin", "main") + git(repo, "reset", "--hard", baseline) + before = (git(repo, "rev-parse", "HEAD"), (repo / ".git/index").read_bytes(), (repo / control).read_bytes()) + with pytest.raises(RuntimeError, match="Bootstrap blobs or modes changed"): + prep.prepare(repo, origin) + assert before == (git(repo, "rev-parse", "HEAD"), (repo / ".git/index").read_bytes(), (repo / control).read_bytes()) + + @pytest.mark.parametrize("selector", ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR", "GIT_OBJECT_DIRECTORY", "GIT_CONFIG_COUNT", "combined"]) def test_selector_child_cannot_inspect_or_mutate_other_repository(producer, selector): diff --git a/tests/test_reddit_source_package.py b/tests/test_reddit_source_package.py index 6f2817c..891fe06 100644 --- a/tests/test_reddit_source_package.py +++ b/tests/test_reddit_source_package.py @@ -14,6 +14,8 @@ from reddit_etl import RedditETL, write_source_package_receipt from exceptions import ETLExtractionError +from reddit_source_package import REDDIT_SCOPE +from scripts import check_reddit_package_admission as gate RUNNER = Path(__file__).resolve().parent.parent / "automation" / "run_reddit_baseline.ps1" POWERSHELL = shutil.which("pwsh") or shutil.which("powershell.exe") @@ -264,6 +266,7 @@ def test_late_publication_failure_restores_local_bytes_without_real_git(tmp_path if ($Label -eq "__FAILING_STEP__" -or "__FAILING_STEP__" -eq "restore") { throw "fixture publication failure" } } function Assert-PreparedMainSha {} +function Assert-LocalRedditAdmission { return ('a' * 64) } function gh { if ("__FAILING_STEP__" -eq "gh pr create") { throw "fixture publication failure" } $global:LASTEXITCODE = 0 @@ -298,7 +301,7 @@ def test_late_publication_failure_restores_local_bytes_without_real_git(tmp_path @pytest.mark.skipif(POWERSHELL is None, reason="PowerShell is required") -@pytest.mark.parametrize("failure", ["add", "commit", "push", "pr"]) +@pytest.mark.parametrize("failure", ["add", "commit", "push", "pr", "index", "coherent", "hook", "hook_index", "hook_package", "success"]) def test_real_local_git_publication_failure_stops_next_run(tmp_path, failure): repo = tmp_path / "producer" remote = tmp_path / "remote.git" @@ -317,29 +320,60 @@ def git(*args, check=True): git("config", "core.hooksPath", str(repo / ".git" / "hooks")) git("remote", "add", "origin", str(remote)) (repo / ".gitignore").write_text("datos/latest/\nautomation/\n", encoding="utf-8") + shutil.copy2(RUNNER.parent.parent / ".gitattributes", repo) data = repo / "datos" data.mkdir() sentiment = data / "reddit_sentimiento_frameworks.csv" topics = data / "reddit_temas_emergentes.csv" receipt = data / "source_packages" / "reddit" / "receipt.json" ignored = data / "latest" / "reddit_temas_emergentes.csv" - sentiment.write_bytes(b"sentiment-before\r\n") + sentiment.write_bytes(b"framework,total_menciones,positivos,neutros,negativos,% positivo,% neutro,% negativo\r\nPython,2,1,1,0,50,50,0\r\n") + sentiment_before = sentiment.read_bytes() topics.write_bytes(b"tema,menciones\nPython,900\n") - git("add", ".gitignore", "datos/reddit_sentimiento_frameworks.csv", "datos/reddit_temas_emergentes.csv") + git("add", ".gitignore", ".gitattributes", "datos/reddit_sentimiento_frameworks.csv", "datos/reddit_temas_emergentes.csv") git("commit", "-qm", "fixture baseline") git("branch", "-M", "main") baseline = git("rev-parse", "HEAD").stdout.strip() automation = repo / "automation" automation.mkdir() shutil.copy2(RUNNER.parent / "reddit_output_transaction.psm1", automation) + package = tmp_path / "package" + package.mkdir() + _csvs(package) + (package / "datos/reddit_sentimiento_frameworks.csv").write_bytes(sentiment_before) + (package / "datos/reddit_temas_emergentes.csv").write_bytes(b"tema,menciones\r\nPython,800\r\n") + finish = datetime.now(timezone.utc).replace(microsecond=0) + write_source_package_receipt(package, finish, finish, finish, list(REDDIT_SCOPE), 800) + replacement = tmp_path / "replacement" + shutil.copytree(package, replacement) + (replacement / "datos/reddit_temas_emergentes.csv").write_bytes(b"tema,menciones\r\nPython,801\r\n") + write_source_package_receipt(replacement, finish, finish, finish, list(REDDIT_SCOPE), 801) hooks = repo / ".git" / "hooks" if failure in {"commit", "push"}: hook = hooks / f"pre-{failure}" hook.write_text("#!/bin/sh\nexit 1\n", encoding="utf-8") os.chmod(hook, 0o700) + if failure in {"hook", "hook_index"}: + hook = hooks / ("pre-commit" if failure == "hook" else "post-commit") + hook.write_text('#!/bin/sh\nprintf "extra" > extra.txt\ngit add extra.txt\n', encoding="utf-8") + os.chmod(hook, 0o700) + if failure == "hook_package": + alternate = package / "alternate.json" + payload = json.loads((package / "datos/source_packages/reddit/receipt.json").read_bytes()) + payload["posts_count"] += 1 + alternate.write_text(json.dumps(payload)) + hook = hooks / "pre-commit" + hook.write_text("#!/bin/sh\ncp '" + alternate.as_posix() + "' datos/source_packages/reddit/receipt.json\ngit add datos/source_packages/reddit/receipt.json\n") + os.chmod(hook, 0o700) runner = RUNNER.read_text(encoding="utf-8") - preamble = runner[: runner.index("Set-Location $repo")] + preamble = runner[: runner.index("Set-Location $repo")].replace( + '(Join-Path $repo "scripts/check_reddit_package_admission.py")', + '"' + str(RUNNER.parent.parent / "scripts/check_reddit_package_admission.py") + '"') + preamble = preamble.replace(' Run-Step "git commit"', + ' Write-Output "ADMITTED_INDEX=$admittedIndex"\n' + ' git ls-files --stage | ForEach-Object { Write-Output "ADMITTED_ENTRY=$_" }\n' + ' Run-Step "git commit"') harness = automation / "publication_fixture.ps1" harness.write_text( preamble @@ -351,35 +385,74 @@ def git(*args, check=True): "datos/source_packages/reddit/receipt.json", "datos/latest/reddit_temas_emergentes.csv" ) -Set-Content -LiteralPath (Join-Path $repo "datos/reddit_sentimiento_frameworks.csv") -Value "sentiment-new" -NoNewline -Set-Content -LiteralPath (Join-Path $repo "datos/reddit_temas_emergentes.csv") -Value "topics-new" -NoNewline +Copy-Item -LiteralPath "__PACKAGE__/datos/reddit_sentimiento_frameworks.csv" -Destination (Join-Path $repo "datos/reddit_sentimiento_frameworks.csv") +Copy-Item -LiteralPath "__PACKAGE__/datos/reddit_temas_emergentes.csv" -Destination (Join-Path $repo "datos/reddit_temas_emergentes.csv") $latest = Join-Path $repo "datos/latest/reddit_temas_emergentes.csv" New-Item -ItemType Directory -Force -Path (Split-Path -Parent $latest) | Out-Null Set-Content -LiteralPath $latest -Value "ignored-new" -NoNewline if ("__FAILURE__" -ne "add") { $receipt = Join-Path $repo "datos/source_packages/reddit/receipt.json" New-Item -ItemType Directory -Force -Path (Split-Path -Parent $receipt) | Out-Null - Set-Content -LiteralPath $receipt -Value "receipt-new" -NoNewline + Copy-Item -LiteralPath "__PACKAGE__/datos/source_packages/reddit/receipt.json" -Destination $receipt +} +if ("__FAILURE__" -eq "index") { + Set-Content -LiteralPath (Join-Path $repo "extra.txt") -Value "extra" + git add extra.txt } function gh { if ("__FAILURE__" -eq "pr") { throw "fixture PR failure" } + Write-Host "ACCEPTED_COMMIT=$((git rev-parse HEAD).Trim())" $global:LASTEXITCODE = 0 return "https://example.invalid/pr" } Set-Location $repo $PreparedMainSha = (git rev-parse HEAD).Trim() -Invoke-SourcePublication -Snapshot $outputSnapshot -'''.replace("__FAILURE__", failure), +$py = "__PYTHON__" +if ("__FAILURE__" -eq "add") { $sourceReceipt = ('a' * 64) } +else { $sourceReceipt = Assert-LocalRedditAdmission -Worktree } +if ("__FAILURE__" -eq "coherent") { + Copy-Item -LiteralPath "__REPLACEMENT__/datos/reddit_temas_emergentes.csv" -Destination (Join-Path $repo "datos/reddit_temas_emergentes.csv") + Copy-Item -LiteralPath "__REPLACEMENT__/datos/source_packages/reddit/receipt.json" -Destination (Join-Path $repo "datos/source_packages/reddit/receipt.json") +} +Invoke-SourcePublication -Snapshot $outputSnapshot -ExpectedReceipt $sourceReceipt +'''.replace("__FAILURE__", failure).replace("__PACKAGE__", package.as_posix()) + .replace("__PYTHON__", os.sys.executable).replace("__REPLACEMENT__", replacement.as_posix()), encoding="utf-8", ) result = subprocess.run( [POWERSHELL, "-NoProfile", "-NonInteractive", "-File", str(harness)], cwd=repo, capture_output=True, text=True, check=False, ) - assert result.returncode != 0, result.stdout + if failure == "success": + assert result.returncode == 0, result.stdout + result.stderr + assert git("branch", "--show-current").stdout.strip() == "main" + assert subprocess.run(["git", "--git-dir", str(remote), "show-ref", "--verify", "--quiet", "refs/heads/reddit-source-fixture"]).returncode == 0 + accepted = next(line.split("=", 1)[1] for line in result.stdout.splitlines() if line.startswith("ACCEPTED_COMMIT=")) + admitted = next(line.split("=", 1)[1] for line in result.stdout.splitlines() if line.startswith("ADMITTED_INDEX=")) + entries = {} + for line in result.stdout.splitlines(): + if line.startswith("ADMITTED_ENTRY="): + metadata, path = line.removeprefix("ADMITTED_ENTRY=").split("\t", 1) + mode, oid, stage = metadata.split() + assert stage == "0" and path.encode() not in entries + entries[path.encode()] = (mode.encode(), oid.encode()) + assert gate.fingerprint(entries) == admitted + assert gate.manifest(repo, accepted) == entries + assert git("rev-parse", "refs/heads/reddit-source-fixture").stdout.strip() == accepted + remote_sha = subprocess.run(["git", "--git-dir", str(remote), "rev-parse", "refs/heads/reddit-source-fixture"], + capture_output=True, text=True, check=True).stdout.strip() + assert remote_sha == accepted + assert git("rev-list", "--parents", "-n", "1", accepted).stdout.split() == [accepted, baseline] + blobs = {path: gate.git(repo, "cat-file", "blob", accepted + ":" + path) for path in gate.PATHS} + assert [blobs[path] for path in gate.PATHS[:2]] == [(package / path).read_bytes() for path in gate.PATHS[:2]] + assert blobs[gate.PATHS[0]] == sentiment_before + assert gate.receipt_identity(blobs[gate.PATHS[2]]) == gate.receipt_identity((package / gate.PATHS[2]).read_bytes()) + assert gate.admit_blobs(repo, baseline, blobs)["eligibility"] == "pass" + return + assert result.returncode != 0, result.stdout + result.stderr reached_steps = [line[4:] for line in result.stdout.splitlines() if line.startswith("==> ")] expected_steps = ["git checkout branch", "git add target files"] - if failure != "add": + if failure not in {"add", "index", "coherent"}: expected_steps.append("git commit") if failure in {"push", "pr"}: expected_steps.append("git push") @@ -388,7 +461,7 @@ def git(*args, check=True): assert "REDDIT_SOURCE_LOCAL_FILES_RESTORED=1" in result.stdout assert "Git state was not rolled back" in result.stdout assert "REDDIT_SOURCE_PUBLICATION_FAILED_ROLLBACK" not in result.stdout - assert sentiment.read_bytes() == b"sentiment-before\r\n" + assert sentiment.read_bytes() == sentiment_before assert topics.read_bytes() == b"tema,menciones\nPython,900\n" assert not receipt.exists() assert not ignored.exists() @@ -415,8 +488,11 @@ def git(*args, check=True): "datos/source_packages/reddit/receipt.json", ]) assert branch == "reddit-source-fixture" - assert sorted(index) == (changed_paths if failure == "commit" else []) - assert sorted(worktree) == (changed_paths if failure != "add" else []) + expected_index = changed_paths[1:] if failure in {"commit", "index", "coherent"} else [] + if failure in {"index", "hook_index"}: + expected_index = sorted([*expected_index, "extra.txt"]) + assert sorted(index) == expected_index + assert sorted(worktree) == (changed_paths[1:] if failure != "add" else []) assert preflight.returncode != 0, "Next scheduled run must not silently retry after publication failure" - assert (head != baseline) is (failure in {"push", "pr"}) + assert (head != baseline) is (failure in {"push", "pr", "hook", "hook_index", "hook_package"}) assert remote_branch is (failure == "pr")