fix(automation): admit exact Reddit package before publication

automation/run_reddit_baseline.ps1

@@ -99,12 +99,14 @@ function Assert-RedditSourcePackage {
 }
 
 function Invoke-SourcePublication {
-  param([Parameter(Mandatory = $true)]$Snapshot)
+  param([Parameter(Mandatory = $true)]$Snapshot, [string]$ExpectedReceipt)
   try {
     Assert-PreparedMainSha
     Run-Step "git checkout branch" @("git", "checkout", "-b", $branch)
     Run-Step "git add target files" (@("git", "add", "--") + $filesToStage)
+    $admittedIndex = Assert-LocalRedditAdmission -Index -ExpectedReceipt $ExpectedReceipt
     Run-Step "git commit" @("git", "commit", "-m", "chore(data): refresh reddit source package $utcDate")
+    Assert-LocalRedditAdmission -ExpectedIndex $admittedIndex -ExpectedReceipt $ExpectedReceipt | Out-Null
     Run-Step "git push" @("git", "push", "-u", "origin", $branch)
     $body = @"
 ## Summary
@@ -139,6 +141,26 @@ function Invoke-SourcePublication {
   }
 }
 
+function Assert-LocalRedditAdmission {
+  param([switch]$Index, [switch]$Worktree, [string]$ExpectedIndex, [string]$ExpectedReceipt)
+  Assert-RedditGitEnvironment
+  $arguments = @("-I", "-B", (Join-Path $repo "scripts/check_reddit_package_admission.py"), "--root", $repo, "--base", $PreparedMainSha)
+  if ($Worktree) { $arguments += "--worktree" }
+  elseif ($Index) { $arguments += @("--index", "--expected-receipt", $ExpectedReceipt) }
+  else {
+    $head = (git rev-parse HEAD).Trim()
+    if ($LASTEXITCODE -ne 0) { throw "Cannot resolve committed package SHA" }
+    $arguments += @("--head", $head, "--expected-index", $ExpectedIndex, "--expected-receipt", $ExpectedReceipt)
+  }
+  $result = @(& $py @arguments)
+  if ($LASTEXITCODE -ne 0 -or $result.Count -ne 1) { throw "Local Reddit package admission failed" }
+  $admission = $result[0] | ConvertFrom-Json
+  $identity = if ($Worktree) { $admission.receipt_fingerprint } else { $admission.index_fingerprint }
+  if ($admission.classification -ne "source-package" -or $admission.eligibility -ne "pass" -or
+      $identity -notmatch '^[0-9a-f]{64}$') { throw "Local admission did not accept a package" }
+  return [string]$identity
+}
+
 Set-Location $repo
 Assert-CleanWorktree
 Assert-PreparedMainSha
@@ -162,6 +184,7 @@ $env:REDDIT_RSS_MAX_ATTEMPTS = "2"
 try {
   Run-Step "reddit source extraction" @($py, "backend\reddit_etl.py", "--source-package")
   Assert-RedditSourcePackage -BaselineMentions $baselineMentions
+  $sourceReceipt = Assert-LocalRedditAdmission -Worktree
   Restore-RedditOutputSnapshotPaths -Snapshot $outputSnapshot -RelativePaths $ignoredOutputs
 }
 catch {
@@ -172,5 +195,5 @@ catch {
   throw $failure
 }
 
-$prUrl = Invoke-SourcePublication -Snapshot $outputSnapshot
+$prUrl = Invoke-SourcePublication -Snapshot $outputSnapshot -ExpectedReceipt $sourceReceipt
 Write-Output "PR_URL=$prUrl"

scripts/check_reddit_package_admission.py

@@ -1,8 +1,10 @@
 """Admit committed Reddit packages as data using trusted code and a real UTC clock."""
 
+import argparse
 import csv
 from datetime import datetime, timezone
 import io
+import hashlib
 import json
 import os
 from pathlib import Path
@@ -14,6 +16,7 @@
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / "backend"))
 from reddit_source_package import producer_minimum_mentions, validate_reddit_source_package  # noqa: E402
+from reddit_source_package import _unique_json_keys  # noqa: E402
 
 PATHS = ("datos/reddit_sentimiento_frameworks.csv", "datos/reddit_temas_emergentes.csv",
          "datos/source_packages/reddit/receipt.json")
@@ -57,6 +60,11 @@ def admit(root, base, head, *, now=None):
         if len(entry) != 4 or entry[:2] != ["100644", "blob"] or entry[3] != path:
             raise ValueError("Package requires three regular 100644 files")
         blobs[path] = git(root, "cat-file", "blob", entry[2])
+    return admit_blobs(root, base, blobs, now=now)
+
+
+def admit_blobs(root, base, blobs, *, now=None):
+    """One provenance and coverage contract for committed and staged bytes."""
     current = now if now is not None else datetime.now(timezone.utc)
     if current.tzinfo is None:
         raise ValueError("Admission clock must be UTC-aware")
@@ -80,6 +88,95 @@ def admit(root, base, head, *, now=None):
             "posts_count_claim": package["posts_count"], "mentions_total": package["mentions_total"]}
 
 
+def manifest(root, head=None):
+    """Capture every path, mode and object; never manufacture a tree from the worktree."""
+    entries = {}
+    raw = git(root, "ls-tree", "-r", "-z", head) if head else git(root, "ls-files", "--stage", "-z")
+    for record in raw.split(b"\0")[:-1]:
+        metadata, path = record.split(b"\t", 1)
+        mode, middle, last = metadata.split()
+        oid, stage = (last, b"0") if head else (middle, last)
+        if stage != b"0" or path in entries:
+            raise ValueError("Unmerged or duplicate index entry")
+        entries[path] = (mode, oid)
+    return entries
+
+
+def fingerprint(entries):
+    return hashlib.sha256(b"".join(path + b"\0" + mode + b" " + oid + b"\0"
+                                   for path, (mode, oid) in sorted(entries.items()))).hexdigest()
+
+
+def receipt_identity(data):
+    receipt = json.loads(data.decode("utf-8"), object_pairs_hook=_unique_json_keys)
+    return hashlib.sha256(json.dumps(receipt, sort_keys=True, separators=(",", ":")).encode()).hexdigest()
+
+
+def admit_worktree(root, base, *, now=None):
+    root = Path(root)
+    base = commit(root, base)
+    blobs = {path: (root / path).read_bytes() for path in PATHS}
+    result = admit_blobs(root, base, blobs, now=now)
+    if git(root, "rev-parse", "HEAD").decode().strip() != base or any((root / path).read_bytes() != data for path, data in blobs.items()):
+        raise ValueError("Source package or HEAD changed during worktree admission")
+    result["receipt_fingerprint"] = receipt_identity(blobs[PATHS[2]])
+    return result
+
+
+def admit_local(root, base, *, head=None, expected_index=None, expected_receipt=None, now=None):
+    """Require a package and preserve the exact admitted complete index across commit hooks."""
+    root = Path(root)
+    base = commit(root, base)
+    before = manifest(root)
+    if head is not None:
+        head = commit(root, head)
+        if git(root, "rev-list", "--parents", "-n", "1", head).decode().split() != [head, base]:
+            raise ValueError("Committed package must have prepared main as its sole parent")
+        entries = manifest(root, head)
+        if not expected_index or fingerprint(entries) != expected_index or before != entries:
+            raise ValueError("Committed tree or index differs from admitted index")
+    else:
+        entries = before
+    baseline = manifest(root, base)
+    changed = {path for path in baseline.keys() | entries.keys() if baseline.get(path) != entries.get(path)}
+    if not changed or not changed <= {path.encode() for path in PATHS} or any(path not in entries for path in changed):
+        raise ValueError("Local publication requires only added or modified package paths")
+    blobs = {}
+    for path in PATHS:
+        mode, oid = entries.get(path.encode(), (None, None))
+        if mode != b"100644":
+            raise ValueError("Package requires three regular 100644 files")
+        blobs[path] = git(root, "cat-file", "blob", oid.decode())
+    result = admit_blobs(root, base, blobs, now=now)
+    if receipt_identity(blobs[PATHS[2]]) != expected_receipt:
+        raise ValueError("Package differs from initially validated source receipt")
+    if manifest(root) != before or git(root, "rev-parse", "HEAD").decode().strip() != (head or base):
+        raise ValueError("Index or HEAD changed during local admission")
+    result["index_fingerprint"] = fingerprint(entries)
+    return result
+
+
+def local_main(argv):
+    parser = argparse.ArgumentParser(description="Read-only local Reddit package admission")
+    parser.add_argument("--root", type=Path, required=True)
+    parser.add_argument("--base", required=True)
+    view = parser.add_mutually_exclusive_group(required=True)
+    view.add_argument("--index", action="store_true")
+    view.add_argument("--worktree", action="store_true")
+    view.add_argument("--head")
+    parser.add_argument("--expected-index")
+    parser.add_argument("--expected-receipt")
+    args = parser.parse_args(argv)
+    try:
+        result = admit_worktree(args.root, args.base) if args.worktree else admit_local(
+            args.root, args.base, head=args.head, expected_index=args.expected_index, expected_receipt=args.expected_receipt)
+        print(json.dumps(result))
+    except (ValueError, OSError, KeyError, UnicodeError, subprocess.TimeoutExpired) as exc:
+        print("Local admission failed: " + str(exc))
+        return 1
+    return 0
+
+
 def event_binding(event, repository):
     pr = event["pull_request"]
     if repository != REPOSITORY or event["repository"]["full_name"] != repository or pr["base"]["ref"] != "main":
@@ -130,4 +227,4 @@ def main():
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    sys.exit(local_main(sys.argv[1:]) if len(sys.argv) > 1 else main())

scripts/prepare_reddit_producer.py

@@ -11,7 +11,8 @@
 
 ORIGIN = "https://github.com/Sam-24-dev/Technology-trend-analysis-platform"
 CONTROLS = ("automation/run_reddit_baseline_guarded.ps1", "automation/run_reddit_baseline.ps1",
-            "automation/reddit_output_transaction.psm1", "scripts/prepare_reddit_producer.py")
+            "automation/reddit_output_transaction.psm1", "scripts/prepare_reddit_producer.py",
+            "scripts/check_reddit_package_admission.py", "backend/reddit_source_package.py")
 GIT_SELECTORS = (r"^GIT_(DIR|WORK_TREE|COMMON_DIR|INDEX_FILE|OBJECT_DIRECTORY|ALTERNATE_OBJECT_DIRECTORIES|"
                  r"NAMESPACE|.*PREFIX|SHALLOW_FILE|GRAFT_FILE|CONFIG.*|CEILING_DIRECTORIES|"
                  r"DISCOVERY_ACROSS_FILESYSTEM|IMPLICIT_WORK_TREE|REPLACE_REF_BASE)$")

tests/test_reddit_package_admission.py

@@ -128,6 +128,130 @@ def test_code_only_skips_historical_package(candidate):
     assert gate.admit(repo, base, save(repo), now=NOW)["eligibility"] == "not-applicable"
 
 
+@pytest.mark.parametrize("newline", ["\n", "\r\n"])
+def test_local_exact_index_and_commit_without_ci(candidate, monkeypatch, newline, capsys):
+    repo, base, receipt = candidate
+    (repo / gate.PATHS[2]).write_bytes((json.dumps(receipt) + newline).encode())
+    original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes())
+    gate.git(repo, "add", "--all")
+    staged = gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW)
+    original = gate.git
+    def local_git(root, *args):
+        assert args[0] not in {"fetch", "push", "remote"}
+        return original(root, *args)
+    monkeypatch.setattr(gate, "git", local_git)
+    monkeypatch.setattr(gate, "publish_check", lambda *args: pytest.fail("Local check publication"))
+    assert original(repo, "rev-parse", base + ":" + gate.PATHS[0]) == original(repo, "rev-parse", ":" + gate.PATHS[0])
+    head = save(repo)
+    assert gate.admit_local(repo, base, head=head, expected_index=staged["index_fingerprint"], expected_receipt=original_receipt, now=NOW) == staged
+    finish = datetime.now(timezone.utc).replace(microsecond=0)
+    write_package(repo, finish=finish)
+    original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes())
+    gate.git(repo, "add", "--all")
+    valid_index = gate.admit_local(repo, head, expected_receipt=original_receipt, now=finish)
+    head = save(repo)
+    expected = valid_index["index_fingerprint"]
+    assert gate.fingerprint(gate.manifest(repo, head)) == expected
+    assert gate.admit(repo, base, head, now=finish)["eligibility"] == "pass"
+    assert gate.receipt_identity(gate.git(repo, "cat-file", "blob", head + ":" + gate.PATHS[2])) == original_receipt
+    assert gate.local_main(["--root", str(repo), "--base", base, "--head", head,
+                           "--expected-index", expected, "--expected-receipt", original_receipt]) == 1
+    assert capsys.readouterr().out.strip() == "Local admission failed: Committed package must have prepared main as its sole parent"
+
+
+@pytest.mark.parametrize("damage", ["empty", "code", "extra", "delete", "rename", "mode", "symlink", "unmerged", "tamper", "duplicate"])
+def test_local_bad_index_is_read_only(candidate, damage):
+    repo, base, _ = candidate
+    original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes())
+    if damage in {"empty", "code"}:
+        gate.git(repo, "restore", "--worktree", "--", *gate.PATHS)
+    if damage in {"code", "extra"}:
+        (repo / "unrelated.py").write_text("pass\n")
+    if damage == "delete":
+        (repo / gate.PATHS[0]).unlink()
+    if damage == "rename":
+        (repo / gate.PATHS[0]).rename(repo / "renamed.csv")
+    if damage == "tamper":
+        (repo / gate.PATHS[1]).write_bytes(b"altered")
+    if damage == "duplicate":
+        path = repo / gate.PATHS[2]
+        path.write_text(path.read_text().replace('"source":', '"source":"duplicate","source":'))
+    gate.git(repo, "add", "--all")
+    oid = gate.git(repo, "rev-parse", base + ":" + gate.PATHS[0]).decode().strip()
+    if damage in {"mode", "symlink"}:
+        gate.git(repo, "update-index", "--cacheinfo", "100755" if damage == "mode" else "120000", oid, gate.PATHS[0])
+    if damage == "unmerged":
+        gate.git(repo, "update-index", "--force-remove", "--", gate.PATHS[0])
+        gate.subprocess.run(["git", "-C", str(repo), "update-index", "--index-info"],
+                            input=f"100644 {oid} 1\t{gate.PATHS[0]}\n".encode(), check=True)
+    before = (gate.git(repo, "show-ref"), (repo / ".git/index").read_bytes())
+    with pytest.raises(ValueError):
+        gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW)
+    assert before == (gate.git(repo, "show-ref"), (repo / ".git/index").read_bytes())
+
+
+@pytest.mark.parametrize("baseline,total,finish", [(300, 399, NOW), (300, 400, NOW), (501, 425, NOW), (501, 426, NOW),
+                                                  (501, 500, NOW + timedelta(seconds=1)), (501, 500, NOW - timedelta(days=1)),
+                                                  (501, 500, NOW - timedelta(days=9))])
+def test_local_utc_and_integer_ceiling(candidate, baseline, total, finish):
+    repo, _, _ = candidate
+    write_package(repo, baseline, NOW - timedelta(days=9))
+    base = save(repo)
+    write_package(repo, total, finish)
+    original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes())
+    gate.git(repo, "add", "--all")
+    if total in {400, 426}:
+        assert gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW)["minimum_mentions"] == total
+    else:
+        with pytest.raises(ValueError):
+            gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW)
+
+
+@pytest.mark.parametrize("committed", [False, True])
+def test_local_valid_but_different_package_is_tampering(candidate, committed):
+    repo, base, _ = candidate
+    original_receipt = gate.receipt_identity((repo / gate.PATHS[2]).read_bytes())
+    gate.git(repo, "add", "--all")
+    admitted = gate.admit_local(repo, base, expected_receipt=original_receipt, now=NOW)["index_fingerprint"]
+    head = save(repo) if not committed else None
+    write_package(repo, 801)
+    gate.git(repo, "add", "--all")
+    head = save(repo) if committed else head
+    with pytest.raises(ValueError, match="differs"):
+        gate.admit_local(repo, base, head=head, expected_index=admitted, expected_receipt=original_receipt, now=NOW)
+
+
+def test_local_cli_real_clock_never_calls_ci(candidate, monkeypatch, capsys):
+    repo, base, _ = candidate
+    finish = datetime.now(timezone.utc).replace(microsecond=0)
+    write_package(repo, finish=finish)
+    gate.git(repo, "add", "--all")
+    monkeypatch.setattr(gate, "main", lambda: pytest.fail("CI entry"))
+    monkeypatch.setattr(gate, "urlopen", lambda *args, **kwargs: pytest.fail("Network"))
+    args = ["--root", str(repo), "--base", base]
+    assert gate.local_main([*args, "--worktree"]) == 0
+    original_receipt = json.loads(capsys.readouterr().out)["receipt_fingerprint"]
+    assert gate.local_main([*args, "--index"]) == 1
+    assert "initially validated source" in capsys.readouterr().out
+    args += ["--expected-receipt", original_receipt]
+    assert gate.local_main([*args, "--head", ""]) == 1
+    assert "exact commit SHA" in capsys.readouterr().out
+    assert gate.local_main([*args, "--index"]) == 0
+    staged = json.loads(capsys.readouterr().out)
+    head = save(repo)
+    assert gate.local_main([*args, "--head", head, "--expected-index", staged["index_fingerprint"]]) == 0
+    assert json.loads(capsys.readouterr().out) == staged
+
+
+def test_local_coherent_staged_replacement_rejects_original_source(candidate):
+    repo, base, _ = candidate
+    original = gate.admit_worktree(repo, base, now=NOW)["receipt_fingerprint"]
+    write_package(repo, 801)
+    gate.git(repo, "add", "--all")
+    with pytest.raises(ValueError, match="initially validated source"):
+        gate.admit_local(repo, base, expected_receipt=original, now=NOW)
+
+
 def test_floor_ceiling_and_verified_baseline(candidate):
     assert (301 * 85 + 99) // 100 == 256 and (300 * 85 + 99) // 100 == 255
     assert producer_minimum_mentions(301) == producer_minimum_mentions(300) == 400

tests/test_reddit_producer_preparation.py

@@ -276,6 +276,21 @@ def test_revision_assertions_precede_baseline_and_publication():
     assert publication.index("Assert-PreparedMainSha") < publication.index('Run-Step "git checkout branch"')
 
 
+@pytest.mark.parametrize("control", ["scripts/check_reddit_package_admission.py", "backend/reddit_source_package.py"])
+def test_preparation_rejects_changed_admission_controls(producer, control):
+    repo, origin, baseline, target = producer
+    assert control in prep.CONTROLS
+    git(repo, "reset", "--hard", target)
+    (repo / control).write_text("changed\n")
+    git(repo, "commit", "-qam", "test: change admission control")
+    git(repo, "push", "-q", "origin", "main")
+    git(repo, "reset", "--hard", baseline)
+    before = (git(repo, "rev-parse", "HEAD"), (repo / ".git/index").read_bytes(), (repo / control).read_bytes())
+    with pytest.raises(RuntimeError, match="Bootstrap blobs or modes changed"):
+        prep.prepare(repo, origin)
+    assert before == (git(repo, "rev-parse", "HEAD"), (repo / ".git/index").read_bytes(), (repo / control).read_bytes())
+
+
 @pytest.mark.parametrize("selector", ["GIT_DIR", "GIT_WORK_TREE", "GIT_INDEX_FILE", "GIT_COMMON_DIR",
                                      "GIT_OBJECT_DIRECTORY", "GIT_CONFIG_COUNT", "combined"])
 def test_selector_child_cannot_inspect_or_mutate_other_repository(producer, selector):