From cb6ea0fbaefce652aeadc726cf68dd0093966188 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 8 Aug 2026 11:47:57 +0100 Subject: [PATCH 1/7] Implement verified ready admission publication --- .../AUTH_HANDOFF.md | 22 + .../STATUS.md | 10 +- ...RT-001-04C2-ready-admission-publication.md | 79 +++- .../0061_submission_bundle_admission.py | 321 +++++++++++++++ backend/app/adapters/artifacts/__init__.py | 118 ++++++ backend/app/interfaces/artifact_operations.py | 12 +- backend/app/modules/artifacts/models.py | 109 +++++ backend/app/modules/artifacts/operator.py | 60 ++- .../modules/artifacts/pre_submit_evidence.py | 17 +- backend/app/modules/artifacts/router.py | 4 + backend/app/modules/artifacts/service.py | 18 +- .../modules/artifacts/submission_admission.py | 36 ++ .../submission_admission_publication.py | 214 ++++++++++ .../artifacts/submission_authorization.py | 49 +++ .../artifacts/submission_materialization.py | 30 +- .../artifacts/submission_preparation.py | 350 ++++++++++++++++ .../app/modules/tasks/pre_submit_context.py | 49 +++ backend/app/modules/tasks/router.py | 84 +++- backend/tests/conftest.py | 4 +- backend/tests/test_alembic.py | 70 +++- backend/tests/test_artifact_architecture.py | 18 + backend/tests/test_artifact_operator_api.py | 12 + .../test_default_pre_submit_execution.py | 175 +++++++- .../tests/test_submission_bundle_admission.py | 384 +++++++++++++++++- docs/architecture_data_model.md | 9 + docs/operations_artifact_storage.md | 13 + docs/spec_artifact_storage_service.md | 43 +- 27 files changed, 2264 insertions(+), 46 deletions(-) create mode 100644 backend/alembic/versions/0061_submission_bundle_admission.py create mode 100644 backend/app/modules/artifacts/submission_admission_publication.py create mode 100644 backend/app/modules/artifacts/submission_preparation.py diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/AUTH_HANDOFF.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/AUTH_HANDOFF.md index 30cbb27e1..f537e363f 100644 --- a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/AUTH_HANDOFF.md +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/AUTH_HANDOFF.md @@ -118,6 +118,28 @@ own facts; ART receives only the typed capability and never imports AUTH-owned repositories. Authorization evidence, capacity reservation, and durable put intent commit atomically before provider I/O. +### Exact 04C2 activation manifest + +The hidden surface is +`POST /api/v1/tasks/{task_id}/submission-bundle-preparations`. Its body is the +one outer ZIP (`Content-Type: application/zip`). It accepts required headers +`X-Task-Assignment-Id`, `Idempotency-Key`, `X-Submission-Summary`, and +`X-Contributor-Attestation`, plus optional `X-Predecessor-Submission-Id`. +Until activation, absent or malformed selectors and authorization denial are +concealed as the same not-found response. The bounded `202` response contains +only `put_attempt_id`, nullable `admission_id`, `status`, and `replayed`. + +The preflight action is exactly `artifact.submission_bundle.prepare`. Final +transaction-bound consumption binds every field of +`SubmissionBundleDurableIntentAuthorityFacts`: actor profile, identity link, +project, task, assignment, nullable predecessor ID/version, passing evidence +set, prepared generation, locked guide/version/source snapshot, effective +artifact policy, checker policy, effective plan, semantic manifest, archive +digest/bytes/media type, storage scheme, operation identity, and nullable +server-selected replay intent. Consumption precedes capacity reservation, +durable intent creation, and provider I/O. AUTH must not accept client-selected +resource facts, serialize handles, or activate any additional action. + ART-05 requires a new human authorization decision for `submission.create` and a separately prepared fixed-service capability for ActionId `artifact.submission.binding.create`, mapped to PermissionId diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/STATUS.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/STATUS.md index 48400f264..b4a335737 100644 --- a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/STATUS.md +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/STATUS.md @@ -132,10 +132,12 @@ exposure, provider I/O, or AUTH activation. `WS-ART-001-04B3` merged through PR #291 as `8f516e6d`. It executes the project-policy continuation through that same plan and sealed tree and persists one immutable platform-plus-project evidence set. AUTH `WS-XINT-002-06A` merged through PR #293 as `1ddb941e` and -activated only the mandatory fixed pre-submit materializer. ART-04C1 then -merged through PR #296 and persists the evidence-linked durable put intent. -The dependency-ordered successor is 04C2; open pull requests determine whether -it is currently under review. +activated only the mandatory fixed pre-submit materializer. ART-04C1 merged +through PR #296 as `6e1d5c9f` and installed the evidence-bound durable put +intent. ART-04C2 is active and owns verified ready-admission publication plus +the still-hidden, fail-closed continuous preparation composition. AUTH action +activation, admission consumption, Submission creation, and binding remain out +of scope. ## Gate diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/chunks/WS-ART-001-04C2-ready-admission-publication.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/chunks/WS-ART-001-04C2-ready-admission-publication.md index c23966a26..0621fb73b 100644 --- a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/chunks/WS-ART-001-04C2-ready-admission-publication.md +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/chunks/WS-ART-001-04C2-ready-admission-publication.md @@ -1,6 +1,6 @@ # Chunk Contract: WS-ART-001-04C2 — Ready Admission Publication -Parent initiative: `WS-ART-001` | Risk: L1 | Status: Proposed after 04C1 +Parent initiative: `WS-ART-001` | Risk: L1 | Status: Active after merged 04C1 ## Goal @@ -9,32 +9,79 @@ verification and compose the hidden continuous contributor endpoint. ## Allowed Files -SubmissionBundleAdmission model/migration/repository, verified -`SubmissionBundleDurableIntent` reload, verification publication integration, -bounded Operator projection, hidden route composition, tests/docs/CI. +Only these implementation surfaces may change: + +- `backend/app/modules/artifacts/models.py` and the next linear Alembic migration; +- the existing canonical `SubmissionBundlePreparationRequest` and port in + `backend/app/interfaces/artifact_operations.py`; +- `backend/app/modules/artifacts/repository.py`; +- one narrow submission-admission publisher module under + `backend/app/modules/artifacts/` and its composition call from + `backend/app/modules/artifacts/service.py`; +- the existing `backend/app/modules/artifacts/pre_submit_evidence.py` only to + derive and persist the admission's database-verifiable locked-context hash; +- the existing hidden submission preparation composition under + `backend/app/modules/artifacts/`, the existing + `backend/app/adapters/artifacts/` composition root, and + `backend/app/modules/tasks/router.py`; TASK-owned locked preparation-plan + projection may change only in + `backend/app/modules/tasks/pre_submit_context.py`; +- bounded admission-usage projection fields in + `backend/app/modules/artifacts/operator.py` and + `backend/app/modules/artifacts/router.py`; +- directly corresponding tests, CI lane inventory, ART specification/data-model + documentation, and this initiative's status/review evidence. ## Not Allowed Changes Submission/binding consumption, public activation, expiry/release/delete, -candidate storage, review/contribution, or new recovery machinery. +candidate storage, review/contribution, or new recovery machinery. Do not +change AUTH catalogue, grants, constraints, evaluators, or action availability; +legacy `TaskService` Submission creation; provider adapters; scanners; recovery +aggregates; or create a second verification path. ## Acceptance Criteria -Only verified matching bytes publish ready. 04C2 locks and reloads the exact +Only verified matching bytes publish `ready`. The generic verifier calls one +narrow submission-admission publisher only from its verified terminal +transaction. The publisher locks and reloads the exact `SubmissionBundleDurableIntent`, its passing/eligible `PreSubmitEvidenceSet`, -generic put attempt, verified content, replica, and receipt as the sole durable -publication lineage; it never parses an opaque request digest or relies on -scratch, prepared handles, or process-local capability state. Lifecycle is -ready->consumed|stale; actor/link/project/task/assignment/predecessor/context/ -manifest/evidence lineage is immutable; abandoned ready remains charged; exact -POST replay returns the same operation/admission; 04A2-04C2 run in one request -with no serialized local handle; fixed pre-submit materializer is active before -later live activation. +generic put attempt, verified `ArtifactContent`, verified `ArtifactReplica`, +and mandatory successful `ArtifactVerificationReceipt` as the sole durable +publication lineage. It also records whichever provider-write evidence exists: +the nullable direct `ArtifactOperationReceipt` or nullable +`ArtifactPutObservationReceipt`. Exactly one of those write-evidence paths must +match the same put attempt and replica lineage. Guide and checker-output +verification remain unaware of submission lifecycle semantics beyond invoking +the typed publisher with durable identifiers. + +The hidden POST composes 04A2-04C1 continuously in the request, returns only a +bounded operation/current-admission result, and never waits for or replaces the +durable verifier. Exact POST replay returns the same durable operation and, when +verification has already published it, the same admission. Ready publication +may therefore occur after the request returns. No scratch path, prepared handle, +byte stream, or process-local capability crosses into the verification job or +publisher. + +04C2 creates only `ready`. Its schema defines the complete +`ready -> consumed|stale` terminal shape, immutable actor/link/project/task/ +assignment/predecessor/context/manifest/evidence lineage, and uniqueness needed +for later consumption, but 04C2 does not execute `consumed` or `stale` +transitions and does not create a Submission or binding. Those mutations and +their final Submission-consumption uniqueness fence belong to 05A. Abandoned +ready admissions remain charged. Fixed pre-submit materializer authority is +already active; contributor preparation remains hidden and +`artifact.submission_bundle.prepare` remains planned/unavailable. ## Verification Commands -Focused verification/publication/lifecycle/concurrency/operator tests, Ruff, -hosted gates, 90% owned subsystem and 78% repository coverage. +Focused verification/publication tests must cover direct acknowledgement, +observed-confirmed recovery, non-verified outcomes, replay, concurrent verified +publication, lineage mismatch, and no guide/checker admission side effect. +Schema tests prove immutable lineage and valid ready/consumed/stale shapes +without exercising 05A transitions. Operator tests prove bounded unbound-ready +and stale counts/bytes. Run Ruff, hosted gates, 90% owned subsystem coverage, +and the 78% repository baseline. ## Required Reviewers diff --git a/backend/alembic/versions/0061_submission_bundle_admission.py b/backend/alembic/versions/0061_submission_bundle_admission.py new file mode 100644 index 000000000..a5a1d820f --- /dev/null +++ b/backend/alembic/versions/0061_submission_bundle_admission.py @@ -0,0 +1,321 @@ +"""Install verified submission-bundle ready admissions. + +Revision ID: 0061_submission_admission +Revises: 0060_submission_bundle_intent +Create Date: 2026-08-08 +""" + +from __future__ import annotations + +import hashlib +import json + +from alembic import op +import sqlalchemy as sa + + +revision = "0061_submission_admission" +down_revision = "0060_submission_bundle_intent" +branch_labels = depends_on = None + + +def upgrade() -> None: + op.add_column( + "pre_submit_evidence_sets", + sa.Column("locked_policy_context_hash", sa.String(71), nullable=True), + ) + bind = op.get_bind() + rows = list( + bind.execute( + sa.text( + "select id,guide_id,guide_version,source_snapshot_id,source_snapshot_sha256," + "locked_guide_sha256,effective_policy_id,locked_artifact_policy_sha256," + "pre_submit_policy_id,locked_checker_policy_sha256,effective_plan_sha256 " + "from pre_submit_evidence_sets" + ) + ).mappings() + ) + # 0058 made evidence rows immutable. This reviewed migration is the sole + # bounded exception: add one deterministic derived column, then restore the + # guard before installing any admission surface. + op.execute( + "alter table pre_submit_evidence_sets disable trigger pre_submit_evidence_sets_immutable" + ) + for row in rows: + value = {key: row[key] for key in row if key != "id"} + encoded = json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + digest = f"sha256:{hashlib.sha256(encoded).hexdigest()}" + bind.execute( + sa.text( + "update pre_submit_evidence_sets set locked_policy_context_hash=:digest " + "where id=:id" + ), + {"id": row["id"], "digest": digest}, + ) + op.execute( + "alter table pre_submit_evidence_sets enable trigger pre_submit_evidence_sets_immutable" + ) + op.alter_column("pre_submit_evidence_sets", "locked_policy_context_hash", nullable=False) + op.create_check_constraint( + "ck_pre_submit_evidence_policy_context_sha256", + "pre_submit_evidence_sets", + "locked_policy_context_hash ~ '^sha256:[0-9a-f]{64}$'", + ) + op.create_table( + "submission_bundle_admissions", + sa.Column("id", sa.String(36), primary_key=True), + sa.Column("durable_intent_id", sa.String(36), nullable=False), + sa.Column("pre_submit_evidence_set_id", sa.String(36), nullable=False), + sa.Column("put_attempt_id", sa.String(36), nullable=False), + sa.Column("artifact_content_id", sa.String(36), nullable=False), + sa.Column("verified_replica_id", sa.String(36), nullable=False), + sa.Column("verification_receipt_id", sa.String(36), nullable=False), + sa.Column("put_operation_receipt_id", sa.String(36)), + sa.Column("put_observation_receipt_id", sa.String(36)), + sa.Column("actor_profile_id", sa.String(36), nullable=False), + sa.Column("identity_link_id", sa.String(36), nullable=False), + sa.Column("project_id", sa.String(36), nullable=False), + sa.Column("task_id", sa.String(36), nullable=False), + sa.Column("assignment_id", sa.String(36), nullable=False), + sa.Column("predecessor_submission_id", sa.String(36)), + sa.Column("predecessor_submission_version", sa.Integer()), + sa.Column("locked_policy_context_hash", sa.String(71), nullable=False), + sa.Column("semantic_manifest_id", sa.String(36), nullable=False), + sa.Column("semantic_manifest_sha256", sa.String(71), nullable=False), + sa.Column("archive_sha256", sa.String(71), nullable=False), + sa.Column("archive_byte_count", sa.BigInteger(), nullable=False), + sa.Column("status", sa.String(16), nullable=False, server_default="ready"), + sa.Column("ready_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("consumed_at", sa.DateTime(timezone=True)), + sa.Column("consumed_by_submission_id", sa.String(36)), + sa.Column("stale_at", sa.DateTime(timezone=True)), + sa.Column("stale_reason", sa.String(500)), + sa.Column( + "created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now() + ), + sa.ForeignKeyConstraint( + ["durable_intent_id"], ["submission_bundle_durable_intents.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["pre_submit_evidence_set_id"], ["pre_submit_evidence_sets.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["put_attempt_id"], ["artifact_put_attempts.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["artifact_content_id"], ["artifact_contents.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["verified_replica_id"], ["artifact_replicas.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["verification_receipt_id"], ["artifact_verification_receipts.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["put_operation_receipt_id"], ["artifact_operation_receipts.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["put_observation_receipt_id"], + ["artifact_put_observation_receipts.id"], + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint(["actor_profile_id"], ["actor_profiles.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint( + ["identity_link_id"], ["actor_identity_links.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["task_id"], ["workstream_tasks.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint(["assignment_id"], ["task_assignments.id"], ondelete="RESTRICT"), + sa.ForeignKeyConstraint( + ["predecessor_submission_id"], ["submissions.id"], ondelete="RESTRICT" + ), + sa.ForeignKeyConstraint( + ["consumed_by_submission_id"], ["submissions.id"], ondelete="RESTRICT" + ), + sa.UniqueConstraint("durable_intent_id", name="uq_submission_bundle_admission_intent"), + sa.UniqueConstraint( + "pre_submit_evidence_set_id", name="uq_submission_bundle_admission_evidence" + ), + sa.UniqueConstraint( + "verification_receipt_id", name="uq_submission_bundle_admission_verification" + ), + sa.CheckConstraint("status in ('ready','consumed','stale')", name="status"), + sa.CheckConstraint( + "locked_policy_context_hash ~ '^sha256:[0-9a-f]{64}$'", name="policy_context_hash" + ), + sa.CheckConstraint( + "semantic_manifest_sha256 ~ '^sha256:[0-9a-f]{64}$'", name="manifest_sha256" + ), + sa.CheckConstraint("archive_sha256 ~ '^sha256:[0-9a-f]{64}$'", name="archive_sha256"), + sa.CheckConstraint("archive_byte_count >= 0", name="archive_size"), + sa.CheckConstraint( + "((put_operation_receipt_id is not null)::int + (put_observation_receipt_id is not null)::int) = 1", + name="write_receipt_shape", + ), + sa.CheckConstraint( + "(predecessor_submission_id is null) = (predecessor_submission_version is null)", + name="predecessor_shape", + ), + sa.CheckConstraint( + "(status='ready' and consumed_at is null and consumed_by_submission_id is null and stale_at is null and stale_reason is null) or " + "(status='consumed' and consumed_at is not null and consumed_by_submission_id is not null and stale_at is null and stale_reason is null) or " + "(status='stale' and consumed_at is null and consumed_by_submission_id is null and stale_at is not null and octet_length(stale_reason) between 1 and 500)", + name="terminal_shape", + ), + ) + for column in ( + "pre_submit_evidence_set_id", + "artifact_content_id", + "actor_profile_id", + "project_id", + "task_id", + "status", + ): + op.create_index( + f"ix_submission_bundle_admissions_{column}", "submission_bundle_admissions", [column] + ) + op.execute( + """ + create function guard_submission_bundle_admission_verified_lineage() + returns trigger language plpgsql as $$ + declare matches integer; + begin + select count(*) into matches + from submission_bundle_durable_intents intent + join pre_submit_evidence_sets evidence + on evidence.id=intent.pre_submit_evidence_set_id + join artifact_put_attempts attempt on attempt.id=intent.put_attempt_id + join artifact_replicas replica on replica.id=attempt.replica_id + join artifact_contents content on content.id=replica.content_id + join artifact_verification_jobs job + on job.originating_put_attempt_id=attempt.id and job.replica_id=replica.id + join artifact_verification_receipts verification + on verification.verification_job_id=job.id + where intent.id=new.durable_intent_id + and evidence.id=new.pre_submit_evidence_set_id + and attempt.id=new.put_attempt_id + and content.id=new.artifact_content_id + and replica.id=new.verified_replica_id + and verification.id=new.verification_receipt_id + and attempt.producer_request_type='submission_bundle' + and attempt.producer_type='actor_profile' + and attempt.producer_ref=evidence.actor_profile_id + and attempt.project_id=evidence.project_id + and attempt.task_id=evidence.task_id + and attempt.media_type='application/zip' + and content.media_type='application/zip' + and attempt.status='object_confirmed' + and evidence.terminal_status='passed' and evidence.eligible + and replica.verification_state='verified' + and replica.availability_state='available' + and replica.integrity_state='valid' + and verification.outcome='verified' + and verification.execution_generation=job.execution_generation + and verification.observed_sha256=attempt.sha256 + and verification.observed_sha256=content.sha256 + and verification.observed_sha256=evidence.archive_sha256 + and verification.observed_byte_count=attempt.byte_count + and verification.observed_byte_count=content.byte_count + and verification.observed_byte_count=evidence.archive_byte_count + and new.actor_profile_id=evidence.actor_profile_id + and new.identity_link_id=evidence.identity_link_id + and new.project_id=evidence.project_id and new.task_id=evidence.task_id + and new.assignment_id=evidence.assignment_id + and new.predecessor_submission_id is not distinct from evidence.predecessor_submission_id + and new.predecessor_submission_version is not distinct from evidence.predecessor_submission_version + and new.locked_policy_context_hash=evidence.locked_policy_context_hash + and new.semantic_manifest_id=evidence.semantic_manifest_id + and new.semantic_manifest_sha256=evidence.semantic_manifest_sha256 + and new.archive_sha256=evidence.archive_sha256 + and new.archive_byte_count=evidence.archive_byte_count + and ((new.put_operation_receipt_id is not null and exists ( + select 1 from artifact_operation_receipts receipt + where receipt.id=new.put_operation_receipt_id + and receipt.put_attempt_id=attempt.id and receipt.replica_id=replica.id + and receipt.outcome='stored_pending_verification')) + or (new.put_observation_receipt_id is not null and exists ( + select 1 from artifact_put_observation_receipts observation + where observation.id=new.put_observation_receipt_id + and observation.put_attempt_id=attempt.id + and observation.outcome='observed_confirmed' + and observation.observed_sha256=attempt.sha256 + and observation.observed_byte_count=attempt.byte_count))); + if matches <> 1 then + raise exception 'submission bundle admission verified lineage mismatch' + using errcode='23514'; + end if; + return new; + end; + $$ + """ + ) + op.execute( + "create trigger submission_bundle_admission_verified_lineage before insert on submission_bundle_admissions for each row execute function guard_submission_bundle_admission_verified_lineage()" + ) + op.execute( + """ + create function guard_submission_bundle_admission_lineage() + returns trigger language plpgsql as $$ + begin + if row(old.durable_intent_id, old.pre_submit_evidence_set_id, old.put_attempt_id, + old.artifact_content_id, old.verified_replica_id, old.verification_receipt_id, + old.put_operation_receipt_id, old.put_observation_receipt_id, + old.actor_profile_id, old.identity_link_id, old.project_id, old.task_id, + old.assignment_id, old.predecessor_submission_id, + old.predecessor_submission_version, + old.locked_policy_context_hash, + old.semantic_manifest_id, old.semantic_manifest_sha256, old.archive_sha256, + old.archive_byte_count, old.ready_at, old.created_at) + is distinct from + row(new.durable_intent_id, new.pre_submit_evidence_set_id, new.put_attempt_id, + new.artifact_content_id, new.verified_replica_id, new.verification_receipt_id, + new.put_operation_receipt_id, new.put_observation_receipt_id, + new.actor_profile_id, new.identity_link_id, new.project_id, new.task_id, + new.assignment_id, new.predecessor_submission_id, + new.predecessor_submission_version, + new.locked_policy_context_hash, + new.semantic_manifest_id, new.semantic_manifest_sha256, new.archive_sha256, + new.archive_byte_count, new.ready_at, new.created_at) + then + raise exception 'submission bundle admission lineage is immutable' using errcode='55000'; + end if; + if old.status <> 'ready' or new.status not in ('consumed','stale') then + raise exception 'invalid submission bundle admission transition' using errcode='23514'; + end if; + return new; + end; + $$ + """ + ) + op.execute( + "create trigger submission_bundle_admission_lineage before update on submission_bundle_admissions for each row execute function guard_submission_bundle_admission_lineage()" + ) + op.execute( + "create function guard_submission_bundle_admission_delete() returns trigger language plpgsql as $$ begin raise exception 'submission bundle admissions cannot be removed' using errcode='55000'; end; $$" + ) + op.execute( + "create trigger submission_bundle_admission_delete before delete or truncate on submission_bundle_admissions for each statement execute function guard_submission_bundle_admission_delete()" + ) + + +def downgrade() -> None: + bind = op.get_bind() + if bind.execute(sa.text("select count(*) from submission_bundle_admissions")).scalar_one(): + raise RuntimeError("cannot remove populated submission-bundle admissions") + op.execute("drop trigger submission_bundle_admission_delete on submission_bundle_admissions") + op.execute("drop function guard_submission_bundle_admission_delete()") + op.execute("drop trigger submission_bundle_admission_lineage on submission_bundle_admissions") + op.execute("drop function guard_submission_bundle_admission_lineage()") + op.execute( + "drop trigger submission_bundle_admission_verified_lineage on submission_bundle_admissions" + ) + op.execute("drop function guard_submission_bundle_admission_verified_lineage()") + op.drop_table("submission_bundle_admissions") + op.drop_constraint( + "ck_pre_submit_evidence_policy_context_sha256", + "pre_submit_evidence_sets", + type_="check", + ) + op.drop_column("pre_submit_evidence_sets", "locked_policy_context_hash") diff --git a/backend/app/adapters/artifacts/__init__.py b/backend/app/adapters/artifacts/__init__.py index 09fdb4a6f..73231930c 100644 --- a/backend/app/adapters/artifacts/__init__.py +++ b/backend/app/adapters/artifacts/__init__.py @@ -27,6 +27,10 @@ ArtifactScratchManager, ) from app.modules.artifacts.submission_archive import SubmissionArchiveLimits +from app.modules.artifacts.submission_authorization import ( + SubmissionBundlePreparationAuthorization, +) +from app.modules.artifacts.submission_preparation import SubmissionBundlePreparationCommand from app.modules.artifacts.schemas import ( ArtifactInternalAuthority, ) @@ -209,6 +213,120 @@ async def runtime(): return PreparedGuideArtifactIngestCommand(service, authority) +def get_submission_bundle_preparation_authorization(): + """Keep contributor preparation fail closed until XINT-05A activation.""" + from app.modules.artifacts.submission_authorization import ( + DenySubmissionBundlePreparationAuthorization, + ) + + return DenySubmissionBundlePreparationAuthorization() + + +def get_submission_bundle_preparation_command( + request: Request, + session: Annotated[AsyncSession, Depends(get_db_session)], + internal_authority: Annotated[ + ArtifactInternalAuthority, + Depends(get_artifact_internal_authority), + ], + authority: Annotated[ + SubmissionBundlePreparationAuthorization, + Depends(get_submission_bundle_preparation_authorization), + ], +) -> SubmissionBundlePreparationCommand: + """Compose the sole hidden contributor preparation path from closed ART ports.""" + from app.modules.artifacts.authorization import ( + PreparedPreSubmitMaterializationAuthorization, + ) + from app.modules.artifacts.service import ( + ArtifactAdmissionService, + ArtifactStorageOrchestrator, + artifact_storage_namespace_spec, + ) + from app.modules.artifacts.submission_admission import SubmissionBundleDurablePutService + from app.modules.artifacts.submission_archive import SubmissionArchiveInspector + from app.modules.artifacts.submission_materialization import ( + PreparedBundleMaterializationService, + PreparedBundlePreSubmitEvidenceService, + ) + from app.modules.artifacts.submission_preparation import ( + PreparedSubmissionBundlePreparationCommand, + SubmissionBundlePreparationRuntime, + ) + + settings = request.app.state.settings + request_id, correlation_id = (UUID(value) for value in request_ids(request)) + + @asynccontextmanager + async def runtime(): + bootstrap = create_artifact_store_bootstrap(settings) + manager = create_artifact_scratch_manager(settings) + materialization_authority = PreparedPreSubmitMaterializationAuthorization( + session, + request_id=request_id, + correlation_id=correlation_id, + ) + try: + namespace = artifact_storage_namespace_spec(settings, bootstrap) + store = bootstrap.initialize_after_namespace_claim( + ArtifactStoreNamespaceClaim( + adapter_identity=bootstrap.identity, + namespace_identity=bootstrap.namespace_identity, + namespace_fingerprint=namespace.namespace_fingerprint, + ) + ) + preparation = ArtifactPreparationService(manager) + catalogue = request.app.state.pre_submission_checker_catalogue + storage_schemes = {"local": "local", "s3_compatible": "s3"} + try: + storage_scheme = storage_schemes[settings.artifact_store_backend] + except KeyError as exc: + raise RuntimeError("unsupported artifact store backend") from exc + materialization = PreparedBundleMaterializationService( + authorization=materialization_authority, + preparation=preparation, + archive_inspector=SubmissionArchiveInspector( + submission_archive_limits(settings) + ), + catalogue=catalogue, + storage_scheme=storage_scheme, + ) + admission = ArtifactAdmissionService(session, settings, namespace) + storage = ArtifactStorageOrchestrator( + session, + store, + namespace, + settings, + internal_authority, + ) + yield SubmissionBundlePreparationRuntime( + preparation=preparation, + inspector=SubmissionArchiveInspector(submission_archive_limits(settings)), + catalogue=catalogue, + materialization=materialization, + evidence=PreparedBundlePreSubmitEvidenceService( + session=session, + materialization=materialization, + ), + durable_put=SubmissionBundleDurablePutService( + session=session, + admission=admission, + storage=storage, + authorization=authority, + ), + ) + finally: + materialization_authority.close() + manager.close() + bootstrap.close() + + return PreparedSubmissionBundlePreparationCommand( + session=session, + authority=authority, + runtime_factory=runtime, + ) + + async def cleanup_stale_artifact_scratch(settings: Settings) -> int: """Run one database-independent stale cleanup with shared construction.""" require_artifact_runtime_eligible(settings) diff --git a/backend/app/interfaces/artifact_operations.py b/backend/app/interfaces/artifact_operations.py index 0b45bfea4..eec9f243c 100644 --- a/backend/app/interfaces/artifact_operations.py +++ b/backend/app/interfaces/artifact_operations.py @@ -7,7 +7,7 @@ from typing import Literal, Protocol from uuid import UUID -from app.modules.artifacts.sources import ArtifactCommitment, PreparedArtifact +from app.modules.artifacts.sources import PreparedArtifact from app.modules.artifacts.submission_archive import SubmissionArchiveInspectionResult from app.modules.artifacts.submission_manifest import ( SubmissionChangeGateResult, @@ -257,13 +257,17 @@ class CheckerOutputBindingRequest: @dataclass(frozen=True, slots=True) class SubmissionBundlePreparationRequest: - """One prepared contributor authority and continuous outer ZIP source.""" + """One authorized continuous contributor ZIP request without local handles.""" - prepared_authorization: PreparedAuthorizationHandle + authorization_context: AuthorizationContext task_id: UUID assignment_id: UUID + predecessor_submission_id: UUID | None + idempotency_key: UUID + summary: str + contributor_attestation: str + media_type: str byte_source: AsyncIterable[bytes] - client_commitment: ArtifactCommitment | None = None @dataclass(frozen=True, slots=True) diff --git a/backend/app/modules/artifacts/models.py b/backend/app/modules/artifacts/models.py index b8755c3d1..d78c7388a 100644 --- a/backend/app/modules/artifacts/models.py +++ b/backend/app/modules/artifacts/models.py @@ -148,6 +148,10 @@ class PreSubmitEvidenceSet(Base): SHA256_CHECK.format(column="locked_checker_policy_sha256"), name="ck_pre_submit_evidence_checker_policy_sha256", ), + CheckConstraint( + SHA256_CHECK.format(column="locked_policy_context_hash"), + name="ck_pre_submit_evidence_policy_context_sha256", + ), CheckConstraint( SHA256_CHECK.format(column="result_manifest_sha256"), name="ck_pre_submit_evidence_result_manifest_sha256", @@ -218,6 +222,7 @@ class PreSubmitEvidenceSet(Base): ForeignKey("pre_submit_checker_policies.id", ondelete="RESTRICT"), nullable=False ) locked_checker_policy_sha256: Mapped[str] = mapped_column(String(71), nullable=False) + locked_policy_context_hash: Mapped[str] = mapped_column(String(71), nullable=False) effective_plan_sha256: Mapped[str] = mapped_column(String(71), nullable=False) catalogue_id: Mapped[str] = mapped_column(String(160), nullable=False) catalogue_version: Mapped[str] = mapped_column(String(40), nullable=False) @@ -1064,6 +1069,110 @@ class SubmissionBundleDurableIntent(Base): ) +class SubmissionBundleAdmission(Base): + """Immutable verified submission-bundle lineage awaiting TASK consumption.""" + + __tablename__ = "submission_bundle_admissions" + __table_args__ = ( + UniqueConstraint("durable_intent_id", name="uq_submission_bundle_admission_intent"), + UniqueConstraint( + "pre_submit_evidence_set_id", name="uq_submission_bundle_admission_evidence" + ), + UniqueConstraint( + "verification_receipt_id", name="uq_submission_bundle_admission_verification" + ), + CheckConstraint("status in ('ready','consumed','stale')", name="status"), + CheckConstraint( + SHA256_CHECK.format(column="locked_policy_context_hash"), + name="policy_context_hash", + ), + CheckConstraint( + SHA256_CHECK.format(column="semantic_manifest_sha256"), name="manifest_sha256" + ), + CheckConstraint(SHA256_CHECK.format(column="archive_sha256"), name="archive_sha256"), + CheckConstraint("archive_byte_count >= 0", name="archive_size"), + CheckConstraint( + "(predecessor_submission_id is null) = (predecessor_submission_version is null)", + name="predecessor_shape", + ), + CheckConstraint( + "((put_operation_receipt_id is not null)::int + " + "(put_observation_receipt_id is not null)::int) = 1", + name="write_receipt_shape", + ), + CheckConstraint( + "(status='ready' and consumed_at is null and consumed_by_submission_id is null " + "and stale_at is null and stale_reason is null) or " + "(status='consumed' and consumed_at is not null and " + "consumed_by_submission_id is not null and stale_at is null and stale_reason is null) or " + "(status='stale' and consumed_at is null and consumed_by_submission_id is null " + "and stale_at is not null and octet_length(stale_reason) between 1 and 500)", + name="terminal_shape", + ), + ) + + id: Mapped[str] = mapped_column(String(36), primary_key=True) + durable_intent_id: Mapped[str] = mapped_column( + ForeignKey("submission_bundle_durable_intents.id", ondelete="RESTRICT"), nullable=False + ) + pre_submit_evidence_set_id: Mapped[str] = mapped_column( + ForeignKey("pre_submit_evidence_sets.id", ondelete="RESTRICT"), nullable=False, index=True + ) + put_attempt_id: Mapped[str] = mapped_column( + ForeignKey("artifact_put_attempts.id", ondelete="RESTRICT"), nullable=False + ) + artifact_content_id: Mapped[str] = mapped_column( + ForeignKey("artifact_contents.id", ondelete="RESTRICT"), nullable=False, index=True + ) + verified_replica_id: Mapped[str] = mapped_column( + ForeignKey("artifact_replicas.id", ondelete="RESTRICT"), nullable=False + ) + verification_receipt_id: Mapped[str] = mapped_column( + ForeignKey("artifact_verification_receipts.id", ondelete="RESTRICT"), nullable=False + ) + put_operation_receipt_id: Mapped[str | None] = mapped_column( + ForeignKey("artifact_operation_receipts.id", ondelete="RESTRICT") + ) + put_observation_receipt_id: Mapped[str | None] = mapped_column( + ForeignKey("artifact_put_observation_receipts.id", ondelete="RESTRICT") + ) + actor_profile_id: Mapped[str] = mapped_column( + ForeignKey("actor_profiles.id", ondelete="RESTRICT"), nullable=False, index=True + ) + identity_link_id: Mapped[str] = mapped_column( + ForeignKey("actor_identity_links.id", ondelete="RESTRICT"), nullable=False + ) + project_id: Mapped[str] = mapped_column( + ForeignKey("projects.id", ondelete="RESTRICT"), nullable=False, index=True + ) + task_id: Mapped[str] = mapped_column( + ForeignKey("workstream_tasks.id", ondelete="RESTRICT"), nullable=False, index=True + ) + assignment_id: Mapped[str] = mapped_column( + ForeignKey("task_assignments.id", ondelete="RESTRICT"), nullable=False + ) + predecessor_submission_id: Mapped[str | None] = mapped_column( + ForeignKey("submissions.id", ondelete="RESTRICT") + ) + predecessor_submission_version: Mapped[int | None] = mapped_column(Integer) + locked_policy_context_hash: Mapped[str] = mapped_column(String(71), nullable=False) + semantic_manifest_id: Mapped[str] = mapped_column(String(36), nullable=False) + semantic_manifest_sha256: Mapped[str] = mapped_column(String(71), nullable=False) + archive_sha256: Mapped[str] = mapped_column(String(71), nullable=False) + archive_byte_count: Mapped[int] = mapped_column(BigInteger, nullable=False) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="ready", index=True) + ready_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + consumed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + consumed_by_submission_id: Mapped[str | None] = mapped_column( + ForeignKey("submissions.id", ondelete="RESTRICT") + ) + stale_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + stale_reason: Mapped[str | None] = mapped_column(String(500)) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, server_default=func.now() + ) + + class ArtifactPutAttemptCharge(Base): """Immutable link from one put attempt to every required scope charge.""" diff --git a/backend/app/modules/artifacts/operator.py b/backend/app/modules/artifacts/operator.py index bc21b860e..50ce3329a 100644 --- a/backend/app/modules/artifacts/operator.py +++ b/backend/app/modules/artifacts/operator.py @@ -5,7 +5,7 @@ from dataclasses import dataclass from uuid import UUID -from sqlalchemy import Select, and_, or_, select +from sqlalchemy import Select, and_, func, or_, select, tuple_ from sqlalchemy.ext.asyncio import AsyncSession from app.core.config import Settings @@ -14,15 +14,19 @@ ArtifactBindingResourceType, ) from app.modules.artifacts.models import ( + ArtifactAdmissionCharge, ArtifactAdmissionScope, ArtifactBinding, ArtifactOperationReceipt, ArtifactPutAttempt, + ArtifactPutAttemptCharge, ArtifactPutObservationReceipt, ArtifactRecoveryAttempt, ArtifactReplica, ArtifactVerificationJob, ArtifactVerificationReceipt, + SubmissionBundleAdmission, + SubmissionBundleDurableIntent, ) from app.modules.artifacts.metrics import ArtifactAdmissionMetrics from app.modules.artifacts.schemas import ( @@ -513,6 +517,48 @@ async def admission_usage( ).all() ) configured = self._configured_limits() + scope_keys = [(row.scope_type, row.scope_id) for row in rows[:limit]] + admission_usage: dict[tuple[str, str, str], tuple[int, int]] = {} + if scope_keys: + usage_rows = await self._session.execute( + select( + ArtifactAdmissionCharge.scope_type, + ArtifactAdmissionCharge.scope_id, + SubmissionBundleAdmission.status, + func.count(SubmissionBundleAdmission.id), + func.coalesce(func.sum(ArtifactAdmissionCharge.byte_count), 0), + ) + .join( + ArtifactPutAttemptCharge, + ArtifactPutAttemptCharge.charge_id == ArtifactAdmissionCharge.id, + ) + .join( + SubmissionBundleDurableIntent, + SubmissionBundleDurableIntent.put_attempt_id + == ArtifactPutAttemptCharge.attempt_id, + ) + .join( + SubmissionBundleAdmission, + SubmissionBundleAdmission.durable_intent_id + == SubmissionBundleDurableIntent.id, + ) + .where( + tuple_( + ArtifactAdmissionCharge.scope_type, + ArtifactAdmissionCharge.scope_id, + ).in_(scope_keys), + SubmissionBundleAdmission.status.in_(("ready", "stale")), + ) + .group_by( + ArtifactAdmissionCharge.scope_type, + ArtifactAdmissionCharge.scope_id, + SubmissionBundleAdmission.status, + ) + ) + admission_usage = { + (scope_type, scope_id, admission_status): (int(count), int(byte_count)) + for scope_type, scope_id, admission_status, count, byte_count in usage_rows + } return self._result( rows, limit, @@ -523,6 +569,18 @@ async def admission_usage( "limit_bytes": row.limit_bytes, "remaining_bytes": row.limit_bytes - row.counted_bytes, "configured_limit_bytes": configured[row.scope_type], + "unbound_ready_count": admission_usage.get( + (row.scope_type, row.scope_id, "ready"), (0, 0) + )[0], + "unbound_ready_bytes": admission_usage.get( + (row.scope_type, row.scope_id, "ready"), (0, 0) + )[1], + "stale_count": admission_usage.get( + (row.scope_type, row.scope_id, "stale"), (0, 0) + )[0], + "stale_bytes": admission_usage.get( + (row.scope_type, row.scope_id, "stale"), (0, 0) + )[1], "cas_version": row.cas_version, "updated_at": row.updated_at, }, diff --git a/backend/app/modules/artifacts/pre_submit_evidence.py b/backend/app/modules/artifacts/pre_submit_evidence.py index df96b8567..1b0b0d948 100644 --- a/backend/app/modules/artifacts/pre_submit_evidence.py +++ b/backend/app/modules/artifacts/pre_submit_evidence.py @@ -438,7 +438,7 @@ def _set_values( key: str(value) if isinstance(value, UUID) else value for key, value in asdict(context).items() } - return { + values = { "operation_identity": operation_identity, **values, "effective_plan_sha256": execution.plan_sha256, @@ -467,6 +467,21 @@ def _set_values( } ), } + values["locked_policy_context_hash"] = canonical_json_hash( + { + "guide_id": values["guide_id"], + "guide_version": values["guide_version"], + "source_snapshot_id": values["source_snapshot_id"], + "source_snapshot_sha256": values["source_snapshot_sha256"], + "locked_guide_sha256": values["locked_guide_sha256"], + "effective_policy_id": values["effective_policy_id"], + "locked_artifact_policy_sha256": values["locked_artifact_policy_sha256"], + "pre_submit_policy_id": values["pre_submit_policy_id"], + "locked_checker_policy_sha256": values["locked_checker_policy_sha256"], + "effective_plan_sha256": values["effective_plan_sha256"], + } + ) + return values class PreSubmitEvidenceService: diff --git a/backend/app/modules/artifacts/router.py b/backend/app/modules/artifacts/router.py index 2a0e52b0e..11014221a 100644 --- a/backend/app/modules/artifacts/router.py +++ b/backend/app/modules/artifacts/router.py @@ -138,6 +138,10 @@ class ArtifactAdmissionUsageResponse(StrictOperatorResponse): limit_bytes: int remaining_bytes: int configured_limit_bytes: int + unbound_ready_count: int + unbound_ready_bytes: int + stale_count: int + stale_bytes: int cas_version: int updated_at: datetime diff --git a/backend/app/modules/artifacts/service.py b/backend/app/modules/artifacts/service.py index a61929a35..820100ff6 100644 --- a/backend/app/modules/artifacts/service.py +++ b/backend/app/modules/artifacts/service.py @@ -1264,7 +1264,7 @@ async def _complete_verification( states ) replica.last_reconciled_at = now - await self._repo.add_verification_receipt( + verification_receipt = await self._repo.add_verification_receipt( ArtifactVerificationReceipt( id=str(uuid4()), verification_job_id=job.id, @@ -1274,6 +1274,15 @@ async def _complete_verification( observed_byte_count=observed_size, ) ) + if outcome == "verified": + from app.modules.artifacts.submission_admission_publication import ( + SubmissionBundleAdmissionPublisher, + ) + + await SubmissionBundleAdmissionPublisher(self._session).publish_verified( + verification_job_id=job.id, + verification_receipt_id=verification_receipt.id, + ) job.status = outcome job.next_run_at = None job.terminal_result_code = outcome @@ -2166,7 +2175,12 @@ async def _submission_bundle_replay_attempt( or attempt is None or attempt.producer_request_type != "submission_bundle" or attempt.status - not in {"prepared", "acknowledgement_unknown", "absent_replay_required"} + not in { + "prepared", + "acknowledgement_unknown", + "absent_replay_required", + "object_confirmed", + } or attempt.sha256 != current.archive_sha256 or attempt.byte_count != current.archive_byte_count or attempt.media_type != current.media_type diff --git a/backend/app/modules/artifacts/submission_admission.py b/backend/app/modules/artifacts/submission_admission.py index 4db1462c3..e0ba009e1 100644 --- a/backend/app/modules/artifacts/submission_admission.py +++ b/backend/app/modules/artifacts/submission_admission.py @@ -43,6 +43,7 @@ class SubmissionBundleDurablePutResult: operation_identity: str status: str replayed: bool + admission_id: UUID | None class SubmissionBundleDurablePutService: @@ -113,6 +114,35 @@ async def publish_after_commit( await prepared.close() raise RuntimeError("submission bundle durable transaction is still active") try: + from app.modules.artifacts.submission_admission_publication import ( + current_submission_bundle_admission_id, + ) + + admission_id = None + if admission.replayed: + async with self._session.begin(): + admission_id = await current_submission_bundle_admission_id( + self._session, + put_attempt_id=admission.attempt_id, + ) + if admission_id is not None: + return SubmissionBundleDurablePutResult( + put_attempt_id=admission.attempt_id, + pre_submit_evidence_set_id=evidence_set_id, + operation_identity=admission.operation_identity, + status="ready", + replayed=True, + admission_id=admission_id, + ) + if admission.replayed and admission.status == "object_confirmed": + return SubmissionBundleDurablePutResult( + put_attempt_id=admission.attempt_id, + pre_submit_evidence_set_id=evidence_set_id, + operation_identity=admission.operation_identity, + status="object_confirmed", + replayed=True, + admission_id=None, + ) if admission.replayed: status = await self._storage.resume_committed_put( attempt_id=admission.attempt_id, @@ -123,12 +153,18 @@ async def publish_after_commit( attempt_id=admission.attempt_id, source=prepared.committed_source, ) + async with self._session.begin(): + admission_id = await current_submission_bundle_admission_id( + self._session, + put_attempt_id=admission.attempt_id, + ) return SubmissionBundleDurablePutResult( put_attempt_id=admission.attempt_id, pre_submit_evidence_set_id=evidence_set_id, operation_identity=admission.operation_identity, status=status, replayed=admission.replayed, + admission_id=admission_id, ) finally: await prepared.close() diff --git a/backend/app/modules/artifacts/submission_admission_publication.py b/backend/app/modules/artifacts/submission_admission_publication.py new file mode 100644 index 000000000..fa04c1fdb --- /dev/null +++ b/backend/app/modules/artifacts/submission_admission_publication.py @@ -0,0 +1,214 @@ +"""Verified-only publication of immutable ready submission-bundle admissions.""" + +from __future__ import annotations + +from uuid import UUID, uuid4 + +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.modules.artifacts.models import ( + ArtifactContent, + ArtifactOperationReceipt, + ArtifactPutAttempt, + ArtifactPutObservationReceipt, + ArtifactReplica, + ArtifactVerificationJob, + ArtifactVerificationReceipt, + PreSubmitEvidenceSet, + SubmissionBundleAdmission, + SubmissionBundleDurableIntent, +) + + +class SubmissionBundleAdmissionPublicationError(RuntimeError): + """Durable verification lineage cannot publish a trusted ready admission.""" + + +class SubmissionBundleAdmissionPublisher: + """Project verified submission bytes into one idempotent ready admission.""" + + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def publish_verified( + self, *, verification_job_id: str, verification_receipt_id: str + ) -> SubmissionBundleAdmission | None: + """Publish only submission-bundle lineage; ignore other producer types.""" + job = await self._session.scalar( + select(ArtifactVerificationJob) + .where(ArtifactVerificationJob.id == verification_job_id) + .with_for_update(key_share=True) + ) + if job is None: + raise SubmissionBundleAdmissionPublicationError("verification job is unavailable") + attempt = await self._session.scalar( + select(ArtifactPutAttempt) + .where(ArtifactPutAttempt.id == job.originating_put_attempt_id) + .with_for_update(key_share=True) + ) + if attempt is None: + raise SubmissionBundleAdmissionPublicationError("put attempt is unavailable") + if attempt.producer_request_type != "submission_bundle": + return None + intent = await self._session.scalar( + select(SubmissionBundleDurableIntent) + .where(SubmissionBundleDurableIntent.put_attempt_id == attempt.id) + # Serialize publication on the immutable intent. A weaker key-share + # lock lets two verifier transactions both miss the admission and + # race on its uniqueness constraint. + .with_for_update() + ) + if intent is None: + raise SubmissionBundleAdmissionPublicationError("durable intent is unavailable") + existing = await self._session.scalar( + select(SubmissionBundleAdmission) + .where(SubmissionBundleAdmission.durable_intent_id == intent.id) + .with_for_update() + ) + if existing is not None: + return existing + evidence = await self._session.scalar( + select(PreSubmitEvidenceSet) + .where(PreSubmitEvidenceSet.id == intent.pre_submit_evidence_set_id) + .with_for_update(key_share=True) + ) + replica = await self._session.scalar( + select(ArtifactReplica) + .where(ArtifactReplica.id == job.replica_id) + .with_for_update(key_share=True) + ) + receipt = await self._session.scalar( + select(ArtifactVerificationReceipt) + .where(ArtifactVerificationReceipt.id == verification_receipt_id) + .with_for_update(key_share=True) + ) + content = ( + await self._session.scalar( + select(ArtifactContent) + .where(ArtifactContent.id == replica.content_id) + .with_for_update(key_share=True) + ) + if replica is not None + else None + ) + if not self._matches_verified_lineage(evidence, attempt, job, replica, content, receipt): + raise SubmissionBundleAdmissionPublicationError( + "verified submission bundle lineage does not match" + ) + operation_receipt_id, observation_receipt_id = await self._write_receipt_ids(attempt) + assert evidence is not None and replica is not None and content is not None + assert receipt is not None + now = await self._session.scalar(select(func.now())) + admission = SubmissionBundleAdmission( + id=str(uuid4()), + durable_intent_id=intent.id, + pre_submit_evidence_set_id=evidence.id, + put_attempt_id=attempt.id, + artifact_content_id=content.id, + verified_replica_id=replica.id, + verification_receipt_id=receipt.id, + put_operation_receipt_id=operation_receipt_id, + put_observation_receipt_id=observation_receipt_id, + actor_profile_id=evidence.actor_profile_id, + identity_link_id=evidence.identity_link_id, + project_id=evidence.project_id, + task_id=evidence.task_id, + assignment_id=evidence.assignment_id, + predecessor_submission_id=evidence.predecessor_submission_id, + predecessor_submission_version=evidence.predecessor_submission_version, + locked_policy_context_hash=evidence.locked_policy_context_hash, + semantic_manifest_id=evidence.semantic_manifest_id, + semantic_manifest_sha256=evidence.semantic_manifest_sha256, + archive_sha256=evidence.archive_sha256, + archive_byte_count=evidence.archive_byte_count, + status="ready", + ready_at=now, + consumed_at=None, + consumed_by_submission_id=None, + stale_at=None, + stale_reason=None, + ) + self._session.add(admission) + await self._session.flush() + return admission + + @staticmethod + def _matches_verified_lineage(evidence, attempt, job, replica, content, receipt) -> bool: + return bool( + evidence is not None + and evidence.terminal_status == "passed" + and evidence.eligible + and replica is not None + and content is not None + and receipt is not None + and receipt.verification_job_id == job.id + and receipt.execution_generation == job.execution_generation + and receipt.outcome == "verified" + and receipt.observed_sha256 + == attempt.sha256 + == content.sha256 + == evidence.archive_sha256 + and receipt.observed_byte_count + == attempt.byte_count + == content.byte_count + == evidence.archive_byte_count + and job.originating_put_attempt_id == attempt.id + and job.replica_id == replica.id + and attempt.replica_id == replica.id + and replica.content_id == content.id + and replica.verification_state == "verified" + and replica.availability_state == "available" + and replica.integrity_state == "valid" + and attempt.project_id == evidence.project_id + and attempt.task_id == evidence.task_id + and attempt.producer_ref == evidence.actor_profile_id + and attempt.media_type == "application/zip" + ) + + async def _write_receipt_ids( + self, attempt: ArtifactPutAttempt + ) -> tuple[str | None, str | None]: + if attempt.receipt_id is not None: + receipt = await self._session.scalar( + select(ArtifactOperationReceipt).where( + ArtifactOperationReceipt.id == attempt.receipt_id, + ArtifactOperationReceipt.put_attempt_id == attempt.id, + ArtifactOperationReceipt.replica_id == attempt.replica_id, + ArtifactOperationReceipt.outcome == "stored_pending_verification", + ) + ) + if receipt is None: + raise SubmissionBundleAdmissionPublicationError("put receipt lineage is invalid") + return receipt.id, None + observation = await self._session.scalar( + select(ArtifactPutObservationReceipt) + .where( + ArtifactPutObservationReceipt.put_attempt_id == attempt.id, + ArtifactPutObservationReceipt.outcome == "observed_confirmed", + ArtifactPutObservationReceipt.observed_sha256 == attempt.sha256, + ArtifactPutObservationReceipt.observed_byte_count == attempt.byte_count, + ) + .order_by(ArtifactPutObservationReceipt.execution_generation.desc()) + .limit(1) + ) + if observation is None: + raise SubmissionBundleAdmissionPublicationError( + "put observation receipt lineage is invalid" + ) + return None, observation.id + + +async def current_submission_bundle_admission_id( + session: AsyncSession, *, put_attempt_id: UUID +) -> UUID | None: + """Return bounded current state for hidden exact request replay.""" + value = await session.scalar( + select(SubmissionBundleAdmission.id) + .join( + SubmissionBundleDurableIntent, + SubmissionBundleDurableIntent.id == SubmissionBundleAdmission.durable_intent_id, + ) + .where(SubmissionBundleDurableIntent.put_attempt_id == str(put_attempt_id)) + ) + return UUID(value) if value is not None else None diff --git a/backend/app/modules/artifacts/submission_authorization.py b/backend/app/modules/artifacts/submission_authorization.py index 5c72774d7..28e2560c9 100644 --- a/backend/app/modules/artifacts/submission_authorization.py +++ b/backend/app/modules/artifacts/submission_authorization.py @@ -2,13 +2,16 @@ from __future__ import annotations +from contextlib import AbstractAsyncContextManager from typing import Protocol +from uuid import UUID from app.modules.artifacts.schemas import ( ArtifactAuthorityDeniedError, SubmissionBundleDurableIntentAuthorityFacts, ) from app.modules.authorization.prepared import PreparedAuthorizationHandle +from app.modules.authorization.runtime import AuthorizationContext class SubmissionBundlePreparedAuthorization(Protocol): @@ -22,6 +25,34 @@ async def consume( ) -> None: ... +class SubmissionBundlePreparationAuthorization(SubmissionBundlePreparedAuthorization, Protocol): + """Own preflight and final transaction-bound contributor preparation authority.""" + + async def preflight( + self, + *, + authorization_context: AuthorizationContext, + task_id: UUID, + assignment_id: UUID, + predecessor_submission_id: UUID | None, + idempotency_key: UUID, + ) -> None: ... + + def transaction(self) -> AbstractAsyncContextManager[object]: ... + + async def prepare_final( + self, + *, + authorization_context: AuthorizationContext, + task_id: UUID, + assignment_id: UUID, + predecessor_submission_id: UUID | None, + idempotency_key: UUID, + ) -> PreparedAuthorizationHandle: ... + + def close(self) -> None: ... + + class DenySubmissionBundlePreparedAuthorization: """Keep contributor durable preparation unavailable until XINT-05A.""" @@ -33,3 +64,21 @@ async def consume( ) -> None: del prepared_authorization, facts raise ArtifactAuthorityDeniedError("submission bundle durable preparation is unavailable") + + +class DenySubmissionBundlePreparationAuthorization(DenySubmissionBundlePreparedAuthorization): + """Keep the complete contributor surface unavailable until XINT-05A.""" + + async def preflight(self, **values: object) -> None: + del values + raise ArtifactAuthorityDeniedError("submission bundle preparation is unavailable") + + def transaction(self) -> AbstractAsyncContextManager[object]: + raise ArtifactAuthorityDeniedError("submission bundle preparation is unavailable") + + async def prepare_final(self, **values: object) -> PreparedAuthorizationHandle: + del values + raise ArtifactAuthorityDeniedError("submission bundle preparation is unavailable") + + def close(self) -> None: + return None diff --git a/backend/app/modules/artifacts/submission_materialization.py b/backend/app/modules/artifacts/submission_materialization.py index af1e88c4d..1fd78a8dd 100644 --- a/backend/app/modules/artifacts/submission_materialization.py +++ b/backend/app/modules/artifacts/submission_materialization.py @@ -323,9 +323,37 @@ async def execute( raise RuntimeError( "pre-submit evidence orchestration requires a transaction-free session" ) + execution = await self.materialize(request) + return await self.persist( + request, + execution=execution, + actor_profile_id=actor_profile_id, + identity_link_id=identity_link_id, + predecessor_submission_id=predecessor_submission_id, + ) + + async def materialize( + self, request: PreparedBundleMaterializationRequest + ) -> PreSubmissionExecutionResult: + """Consume fixed-service authority while its owning transaction is active.""" + return await self._materialization.materialize_prepared_bundle(request) + + async def persist( + self, + request: PreparedBundleMaterializationRequest, + *, + execution: PreSubmissionExecutionResult, + actor_profile_id: UUID, + identity_link_id: UUID, + predecessor_submission_id: UUID | None, + ) -> PreSubmitEvidencePersistenceResult: + """Persist completed, cleaned execution evidence in a fresh transaction.""" + if self._session.in_transaction(): + raise RuntimeError( + "pre-submit evidence persistence requires a transaction-free session" + ) commitment = request.prepared_artifact.commitment prepared_generation_id = request.prepared_artifact.generation_id - execution = await self._materialization.materialize_prepared_bundle(request) async with self._session.begin(): await self._session.execute(text("set transaction isolation level read committed")) return await PreSubmitEvidenceService(self._session).persist( diff --git a/backend/app/modules/artifacts/submission_preparation.py b/backend/app/modules/artifacts/submission_preparation.py new file mode 100644 index 000000000..8504c0d9f --- /dev/null +++ b/backend/app/modules/artifacts/submission_preparation.py @@ -0,0 +1,350 @@ +"""Hidden continuous contributor ZIP preparation orchestration.""" + +from __future__ import annotations + +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager +from dataclasses import dataclass +from typing import Protocol +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.interfaces.artifact_operations import ( + PreparedBundleMaterializationRequest, + SubmissionBundlePreparationPort, + SubmissionBundlePreparationRequest, +) +from app.modules.artifacts.models import ( + ArtifactPutAttempt, + PreSubmitEvidenceSet, + SubmissionBundleAdmission, + SubmissionBundleDurableIntent, +) +from app.modules.artifacts.preparation import ArtifactPreparationService +from app.modules.artifacts.submission_admission import ( + SubmissionBundleDurablePutRequest, + SubmissionBundleDurablePutResult, + SubmissionBundleDurablePutService, +) +from app.modules.artifacts.submission_archive import SubmissionArchiveInspector +from app.modules.artifacts.submission_authorization import ( + SubmissionBundlePreparationAuthorization, +) +from app.modules.artifacts.submission_manifest import ( + SubmissionCanonicalPredecessor, + build_submission_manifest, + evaluate_submission_change, +) +from app.modules.artifacts.submission_materialization import ( + PreparedBundleMaterializationService, + PreparedBundlePreSubmitEvidenceService, +) +from app.modules.checkers.catalogue import PreSubmissionCheckerCatalogue +from app.modules.checkers.pre_submit_execution import SubmissionPacketView +from app.modules.tasks.pre_submit_context import ( + compile_locked_pre_submit_plan, + load_canonical_submission_version, + load_locked_pre_submit_context, +) + + +class SubmissionBundlePreparationRejected(RuntimeError): + """The complete effective pre-submit execution did not produce passing custody.""" + + +@dataclass(frozen=True, slots=True) +class SubmissionBundlePreparationResult: + put_attempt_id: UUID + admission_id: UUID | None + status: str + replayed: bool + + +@dataclass(frozen=True, slots=True) +class SubmissionBundlePreparationRuntime: + preparation: ArtifactPreparationService + inspector: SubmissionArchiveInspector + catalogue: PreSubmissionCheckerCatalogue + materialization: PreparedBundleMaterializationService + evidence: PreparedBundlePreSubmitEvidenceService + durable_put: SubmissionBundleDurablePutService + + +class SubmissionBundlePreparationCommand(SubmissionBundlePreparationPort, Protocol): + async def prepare( + self, request: SubmissionBundlePreparationRequest + ) -> SubmissionBundlePreparationResult: ... + + +class PreparedSubmissionBundlePreparationCommand: + """Keep every process-local capability within one hidden request.""" + + def __init__( + self, + *, + session: AsyncSession, + authority: SubmissionBundlePreparationAuthorization, + runtime_factory: Callable[ + [], AbstractAsyncContextManager[SubmissionBundlePreparationRuntime] + ], + ) -> None: + self._session = session + self._authority = authority + self._runtime_factory = runtime_factory + + async def prepare( + self, request: SubmissionBundlePreparationRequest + ) -> SubmissionBundlePreparationResult: + if type(request) is not SubmissionBundlePreparationRequest: + raise TypeError("invalid submission bundle preparation request") + prepared = None + try: + await self._authority.preflight( + authorization_context=request.authorization_context, + task_id=request.task_id, + assignment_id=request.assignment_id, + predecessor_submission_id=request.predecessor_submission_id, + idempotency_key=request.idempotency_key, + ) + if request.media_type.partition(";")[0].strip().lower() != "application/zip": + raise SubmissionBundlePreparationRejected("submission_bundle_media_type_invalid") + async with self._runtime_factory() as runtime: + async with self._session.begin(): + locked = await load_locked_pre_submit_context( + self._session, + actor_profile_id=request.authorization_context.actor_profile_id, + identity_link_id=request.authorization_context.identity_link_id, + task_id=request.task_id, + assignment_id=request.assignment_id, + predecessor_submission_id=request.predecessor_submission_id, + include_actor_identity_locks=False, + ) + plan = compile_locked_pre_submit_plan(locked, runtime.catalogue) + predecessor = await self._load_predecessor(request.predecessor_submission_id) + prepared = await runtime.preparation.prepare( + request.byte_source, + media_type="application/zip", + ) + async with self._session.begin(): + materialization_handle = await runtime.materialization.prepare_authorization( + task_id=request.task_id, + assignment_id=request.assignment_id, + submission_artifact_policy_id=locked.effective_policy_id, + checker_policy_id=locked.pre_submit_policy_id, + prepared_artifact=prepared, + effective_plan=plan, + idempotency_key=request.idempotency_key, + ) + inspection = await prepared.inspect(runtime.inspector) + manifest = build_submission_manifest(inspection) + change_gate = evaluate_submission_change( + commitment=prepared.commitment, + manifest=manifest, + predecessor=predecessor, + predecessor_exists=request.predecessor_submission_id is not None, + current_predecessor=predecessor, + ) + materialization_request = PreparedBundleMaterializationRequest( + prepared_authorization=materialization_handle, + task_id=request.task_id, + assignment_id=request.assignment_id, + submission_artifact_policy_id=locked.effective_policy_id, + checker_policy_id=locked.pre_submit_policy_id, + prepared_artifact=prepared, + effective_plan=plan, + inspection=inspection, + manifest=manifest, + change_gate=change_gate, + packet=SubmissionPacketView( + summary=request.summary, + contributor_attestation=request.contributor_attestation, + ), + ) + execution = await runtime.evidence.materialize(materialization_request) + evidence = await runtime.evidence.persist( + materialization_request, + execution=execution, + actor_profile_id=request.authorization_context.actor_profile_id, + identity_link_id=request.authorization_context.identity_link_id, + predecessor_submission_id=request.predecessor_submission_id, + ) + if evidence.pass_capability is None: + replay = await self._existing_durable_result(evidence.evidence.evidence_set_id) + if replay is None: + raise SubmissionBundlePreparationRejected( + "pre_submission_checked_custody_unavailable" + ) + await prepared.close() + prepared = None + return replay + replay_intent_id = await self._matching_replay_intent( + evidence.evidence.evidence_set_id + ) + async with self._authority.transaction(): + final_handle = await self._authority.prepare_final( + authorization_context=request.authorization_context, + task_id=request.task_id, + assignment_id=request.assignment_id, + predecessor_submission_id=request.predecessor_submission_id, + idempotency_key=request.idempotency_key, + ) + retained, _, durable = await runtime.durable_put.admit_in_transaction( + SubmissionBundleDurablePutRequest( + prepared_authorization=final_handle, + prepared_artifact=prepared, + pass_capability=evidence.pass_capability, + replay_durable_intent_id=replay_intent_id, + ) + ) + prepared = None + result = await runtime.durable_put.publish_after_commit( + retained, + evidence.evidence.evidence_set_id, + durable, + ) + return self._result(result) + finally: + if prepared is not None: + await prepared.close() + self._authority.close() + + async def _matching_replay_intent(self, evidence_id: UUID) -> UUID | None: + """Find an older exact lineage without trusting client replay selectors.""" + async with self._session.begin(): + current = await self._session.get(PreSubmitEvidenceSet, str(evidence_id)) + if current is None: + raise SubmissionBundlePreparationRejected( + "pre_submission_checked_custody_unavailable" + ) + fields = ( + "actor_profile_id", + "identity_link_id", + "project_id", + "task_id", + "assignment_id", + "predecessor_submission_id", + "predecessor_submission_version", + "archive_sha256", + "archive_byte_count", + "semantic_manifest_sha256", + "guide_id", + "guide_version", + "source_snapshot_id", + "source_snapshot_sha256", + "locked_guide_sha256", + "effective_policy_id", + "locked_artifact_policy_sha256", + "pre_submit_policy_id", + "locked_checker_policy_sha256", + "effective_plan_sha256", + "catalogue_id", + "catalogue_version", + "catalogue_manifest_sha256", + "storage_scheme", + "terminal_status", + "eligible", + "result_count", + "result_manifest_sha256", + ) + statement = ( + select(SubmissionBundleDurableIntent.id) + .join( + PreSubmitEvidenceSet, + PreSubmitEvidenceSet.id + == SubmissionBundleDurableIntent.pre_submit_evidence_set_id, + ) + .join( + ArtifactPutAttempt, + ArtifactPutAttempt.id == SubmissionBundleDurableIntent.put_attempt_id, + ) + .where( + PreSubmitEvidenceSet.id != current.id, + ArtifactPutAttempt.status.in_( + ( + "prepared", + "acknowledgement_unknown", + "absent_replay_required", + "object_confirmed", + ) + ), + *( + getattr(PreSubmitEvidenceSet, field) == getattr(current, field) + for field in fields + ), + ) + .order_by(SubmissionBundleDurableIntent.created_at) + .limit(1) + ) + value = await self._session.scalar(statement) + return UUID(value) if value is not None else None + + async def _existing_durable_result( + self, evidence_id: UUID + ) -> SubmissionBundlePreparationResult | None: + async with self._session.begin(): + row = ( + await self._session.execute( + select( + SubmissionBundleDurableIntent, + ArtifactPutAttempt, + SubmissionBundleAdmission, + ) + .join( + ArtifactPutAttempt, + ArtifactPutAttempt.id == SubmissionBundleDurableIntent.put_attempt_id, + ) + .outerjoin( + SubmissionBundleAdmission, + SubmissionBundleAdmission.durable_intent_id + == SubmissionBundleDurableIntent.id, + ) + .where( + SubmissionBundleDurableIntent.pre_submit_evidence_set_id == str(evidence_id) + ) + ) + ).one_or_none() + if row is None: + return None + _, attempt, admission = row + return SubmissionBundlePreparationResult( + put_attempt_id=UUID(attempt.id), + admission_id=UUID(admission.id) if admission is not None else None, + status="ready" if admission is not None else attempt.status, + replayed=True, + ) + + async def _load_predecessor( + self, submission_id: UUID | None + ) -> SubmissionCanonicalPredecessor | None: + if submission_id is None: + return None + admission = await self._session.scalar( + select(SubmissionBundleAdmission).where( + SubmissionBundleAdmission.consumed_by_submission_id == str(submission_id), + SubmissionBundleAdmission.status == "consumed", + ) + ) + version = await load_canonical_submission_version( + self._session, submission_id=submission_id + ) + if admission is None or version is None: + raise SubmissionBundlePreparationRejected( + "submission_canonical_predecessor_unavailable" + ) + return SubmissionCanonicalPredecessor( + submission_id=submission_id, + submission_version=version, + archive_sha256=admission.archive_sha256, + semantic_manifest_sha256=admission.semantic_manifest_sha256, + ) + + @staticmethod + def _result(result: SubmissionBundleDurablePutResult) -> SubmissionBundlePreparationResult: + return SubmissionBundlePreparationResult( + put_attempt_id=result.put_attempt_id, + admission_id=result.admission_id, + status=result.status, + replayed=result.replayed, + ) diff --git a/backend/app/modules/tasks/pre_submit_context.py b/backend/app/modules/tasks/pre_submit_context.py index 0145a262f..88224531c 100644 --- a/backend/app/modules/tasks/pre_submit_context.py +++ b/backend/app/modules/tasks/pre_submit_context.py @@ -9,6 +9,12 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.core.hashing import canonical_json_hash +from app.modules.checkers.catalogue import PreSubmissionCheckerCatalogue +from app.modules.checkers.effective_plan import ( + EffectivePreSubmissionExecutionPlan, + EffectivePreSubmissionPlanLineage, + compile_effective_pre_submission_execution_plan, +) from app.modules.actors.models import ActorIdentityLink, ActorProfile from app.modules.projects.models import ( EffectiveProjectSubmissionArtifactPolicy, @@ -41,6 +47,8 @@ class LockedPreSubmitContext: effective_policy_sha256: str pre_submit_policy_id: UUID pre_submit_policy_sha256: str + effective_policy: dict[str, object] + compiled_pre_submit_bundle: dict[str, object] async def load_locked_pre_submit_context( @@ -175,4 +183,45 @@ async def load_locked_pre_submit_context( effective_policy_sha256=effective_policy.effective_policy_hash, pre_submit_policy_id=UUID(checker_policy.id), pre_submit_policy_sha256=checker_policy.compiled_bundle_hash, + effective_policy=dict(effective_policy.effective_policy), + compiled_pre_submit_bundle=dict(checker_policy.compiled_bundle), + ) + + +def compile_locked_pre_submit_plan( + context: LockedPreSubmitContext, + catalogue: PreSubmissionCheckerCatalogue, +) -> EffectivePreSubmissionExecutionPlan: + """Compile one exact plan from TASK-locked policy rows and the fixed catalogue.""" + guide_version_text = context.guide_version + if guide_version_text.startswith("v"): + guide_version_text = guide_version_text[1:] + try: + guide_version = int(guide_version_text) + except ValueError as exc: + raise PreSubmitLockedContextInvalid("pre_submit_guide_version_invalid") from exc + return compile_effective_pre_submission_execution_plan( + lineage=EffectivePreSubmissionPlanLineage( + project_id=context.project_id, + guide_id=context.guide_id, + guide_version=guide_version, + source_snapshot_id=context.source_snapshot_id, + source_snapshot_hash=context.source_snapshot_sha256, + effective_policy_id=context.effective_policy_id, + effective_policy_hash=context.effective_policy_sha256, + pre_submit_policy_id=context.pre_submit_policy_id, + pre_submit_policy_bundle_hash=context.pre_submit_policy_sha256, + ), + effective_policy=context.effective_policy, + compiled_bundle=context.compiled_pre_submit_bundle, + catalogue=catalogue, + ) + + +async def load_canonical_submission_version( + session: AsyncSession, *, submission_id: UUID +) -> int | None: + """Project TASK-owned immutable Submission version without leaking its model.""" + return await session.scalar( + select(Submission.version).where(Submission.id == str(submission_id)) ) diff --git a/backend/app/modules/tasks/router.py b/backend/app/modules/tasks/router.py index cef52478a..fb3ebe710 100644 --- a/backend/app/modules/tasks/router.py +++ b/backend/app/modules/tasks/router.py @@ -3,15 +3,26 @@ from __future__ import annotations from typing import Annotated +from uuid import UUID -from fastapi import APIRouter, Depends, HTTPException, Request +from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from fastapi.responses import JSONResponse +from pydantic import BaseModel, ConfigDict from sqlalchemy.ext.asyncio import AsyncSession from app.api.deps.auth import actor_registry_http_error, get_registered_actor from app.core.api_controls import StructuredHTTPException, error_response from app.core.permissions import PermissionDenied from app.db.session import get_db_session +from app.adapters.artifacts import get_submission_bundle_preparation_command +from app.interfaces.artifact_operations import SubmissionBundlePreparationRequest +from app.modules.artifacts.authorization import get_artifact_authorization_context +from app.modules.artifacts.schemas import ArtifactAuthorityDeniedError +from app.modules.artifacts.submission_preparation import ( + SubmissionBundlePreparationCommand, + SubmissionBundlePreparationRejected, +) +from app.modules.authorization.runtime import AuthorizationContext from app.modules.actors.schemas import ( LegacyWorkflowEligibilityActivationRequest, LegacyWorkflowEligibilityResponse, @@ -34,6 +45,18 @@ router = APIRouter(tags=["tasks"]) + +class SubmissionBundlePreparationResponse(BaseModel): + """Bounded hidden operation state without provider or scratch coordinates.""" + + model_config = ConfigDict(extra="forbid", from_attributes=True) + + put_attempt_id: UUID + admission_id: UUID | None + status: str + replayed: bool + + CANONICAL_ERROR_OBJECT_SCHEMA = {"$ref": "#/components/schemas/ApiError"} PRE_SUBMIT_DOMAIN_ERROR_RESPONSE_SCHEMA = { "oneOf": [ @@ -368,6 +391,65 @@ async def start_task( raise task_http_error(exc) from exc +@router.post( + "/tasks/{task_id}/submission-bundle-preparations", + response_model=SubmissionBundlePreparationResponse, + status_code=status.HTTP_202_ACCEPTED, + include_in_schema=False, +) +async def prepare_submission_bundle( + task_id: str, + request: Request, + context: Annotated[AuthorizationContext, Depends(get_artifact_authorization_context)], + command: Annotated[ + SubmissionBundlePreparationCommand, + Depends(get_submission_bundle_preparation_command), + ], + assignment_id: Annotated[str | None, Header(alias="X-Task-Assignment-Id")] = None, + idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, + summary: Annotated[str | None, Header(alias="X-Submission-Summary")] = None, + contributor_attestation: Annotated[ + str | None, Header(alias="X-Contributor-Attestation") + ] = None, + predecessor_submission_id: Annotated[ + str | None, Header(alias="X-Predecessor-Submission-Id") + ] = None, +) -> SubmissionBundlePreparationResponse: + """Run the hidden continuous ZIP preparation surface; AUTH remains fail closed.""" + if None in (assignment_id, idempotency_key, summary, contributor_attestation): + raise HTTPException(status_code=404, detail="Task not found") + assert assignment_id is not None and idempotency_key is not None + assert summary is not None and contributor_attestation is not None + try: + identifiers = ( + UUID(task_id), + UUID(assignment_id), + UUID(idempotency_key), + UUID(predecessor_submission_id) if predecessor_submission_id else None, + ) + except ValueError as exc: + raise HTTPException(status_code=404, detail="Task not found") from exc + try: + result = await command.prepare( + SubmissionBundlePreparationRequest( + authorization_context=context, + task_id=identifiers[0], + assignment_id=identifiers[1], + predecessor_submission_id=identifiers[3], + idempotency_key=identifiers[2], + summary=summary, + contributor_attestation=contributor_attestation, + media_type=request.headers.get("content-type", ""), + byte_source=request.stream(), + ) + ) + except ArtifactAuthorityDeniedError as exc: + raise HTTPException(status_code=404, detail="Task not found") from exc + except SubmissionBundlePreparationRejected as exc: + raise HTTPException(status_code=422, detail=str(exc)) from exc + return SubmissionBundlePreparationResponse.model_validate(result, from_attributes=True) + + @router.post( "/tasks/{task_id}/submissions", response_model=SubmissionResponse, diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index a996952ad..672d3ef6e 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -21,7 +21,7 @@ from scripts.run_isolated_tests import LOOPBACK, NAME_RE, ROLE_RE DDL_LOCK_DIRECTORY = Path("/tmp") -EXPECTED_PUBLIC_SCHEMA_SHA256 = "5f2f2345f26361ba82bb8da2a8ee098223b4726d39070df58b704387284c3a1e" +EXPECTED_PUBLIC_SCHEMA_SHA256 = "c52b16256a213cfc03827fd8f8107420cf0571dbf8ca839c3d8ff1b1f564ed60" PROTECTED_TEST_TABLES = ( "actor_profile_migration_state", "alembic_version", @@ -94,6 +94,7 @@ "review_queue_entries", "submission_policy_mutation_idempotency_records", "submission_artifact_policies", + "submission_bundle_admissions", "submission_bundle_durable_intents", "submissions", "task_assignments", @@ -124,6 +125,7 @@ "review_queue_entries", "review_policies", "revision_policies", + "submission_bundle_admissions", "submission_bundle_durable_intents", "submission_policy_mutation_idempotency_records", ) diff --git a/backend/tests/test_alembic.py b/backend/tests/test_alembic.py index 23346b68e..a78b27e68 100644 --- a/backend/tests/test_alembic.py +++ b/backend/tests/test_alembic.py @@ -73,7 +73,7 @@ snapshot_existing_service_rows, ) -HEAD_REVISION = "0060_submission_bundle_intent" +HEAD_REVISION = "0061_submission_admission" pytestmark = pytest.mark.postgres_schema_contract @@ -292,6 +292,74 @@ def test_0060_submission_bundle_intent_empty_round_trip( assert "artifact_receipt_producer_reference" in installed["receipt_triggers"] +async def _submission_bundle_admission_schema(database_url: str) -> dict[str, object]: + engine = create_async_engine(database_url) + try: + async with engine.connect() as connection: + table_exists = bool( + await connection.scalar( + text("select to_regclass('submission_bundle_admissions') is not null") + ) + ) + constraints = set( + ( + await connection.scalars( + text( + "select conname from pg_constraint where conrelid=" + "to_regclass('submission_bundle_admissions')" + ) + ) + ).all() + ) + triggers = set( + ( + await connection.scalars( + text( + "select tgname from pg_trigger where not tgisinternal and " + "tgrelid=to_regclass('submission_bundle_admissions')" + ) + ) + ).all() + ) + return {"table_exists": table_exists, "constraints": constraints, "triggers": triggers} + finally: + await engine.dispose() + + +def test_0061_submission_bundle_admission_empty_round_trip( + isolated_database_env: str, + migration_lock, +) -> None: + config = _alembic_config() + with migration_lock(): + try: + command.downgrade(config, "0060_submission_bundle_intent") + prior = asyncio.run(_submission_bundle_admission_schema(isolated_database_env)) + command.upgrade(config, HEAD_REVISION) + installed = asyncio.run(_submission_bundle_admission_schema(isolated_database_env)) + command.downgrade(config, "0060_submission_bundle_intent") + restored = asyncio.run(_submission_bundle_admission_schema(isolated_database_env)) + command.upgrade(config, HEAD_REVISION) + repeated = asyncio.run(_submission_bundle_admission_schema(isolated_database_env)) + finally: + command.upgrade(config, "head") + + assert prior == restored == {"table_exists": False, "constraints": set(), "triggers": set()} + assert installed == repeated + assert { + "uq_submission_bundle_admission_intent", + "uq_submission_bundle_admission_evidence", + "uq_submission_bundle_admission_verification", + "ck_submission_bundle_admissions_terminal_shape", + "ck_submission_bundle_admissions_write_receipt_shape", + }.issubset(installed["constraints"]) + assert installed["triggers"] == { + "submission_bundle_admission_verified_lineage", + "submission_bundle_admission_lineage", + "submission_bundle_admission_delete", + } + + async def _submission_policy_authority_shape(database_url: str) -> dict[str, object]: engine = create_async_engine(database_url) try: diff --git a/backend/tests/test_artifact_architecture.py b/backend/tests/test_artifact_architecture.py index abdca7b47..f1548906b 100644 --- a/backend/tests/test_artifact_architecture.py +++ b/backend/tests/test_artifact_architecture.py @@ -58,6 +58,7 @@ "ArtifactRecoveryRequest", "GuideSufficiencyMaterialRequest", "GuideSourceMaterializationRequest", + "SubmissionBundlePreparationRequest", } PREPARED_HANDLE_FORBIDDEN_ROOTS = ( APP_ROOT / "adapters", @@ -565,6 +566,23 @@ def test_durable_artifact_mutation_ports_require_process_local_prepared_authorit assert "AuthorizationContext" not in _declared_annotation_names(node) +def test_submission_preparation_http_request_never_carries_prepared_authority() -> None: + tree = _tree(ARTIFACT_OPERATIONS) + request_class = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == "SubmissionBundlePreparationRequest" + ) + fields = { + node.target.id: _annotation_names(node.annotation) + for node in request_class.body + if isinstance(node, ast.AnnAssign) and isinstance(node.target, ast.Name) + } + assert "prepared_authorization" not in fields + assert fields["authorization_context"] == {"AuthorizationContext"} + assert fields["idempotency_key"] == {"UUID"} + + def test_prepared_handle_never_enters_public_async_or_provider_contracts() -> None: violations: list[str] = [] schema_files = tuple(APP_ROOT.glob("modules/**/schemas.py")) diff --git a/backend/tests/test_artifact_operator_api.py b/backend/tests/test_artifact_operator_api.py index 22d6cba68..517b53054 100644 --- a/backend/tests/test_artifact_operator_api.py +++ b/backend/tests/test_artifact_operator_api.py @@ -433,6 +433,18 @@ async def context_override(): ) assert usage.status_code == 200 assert usage.json()["items"] + for item in usage.json()["items"]: + assert { + "unbound_ready_count": item["unbound_ready_count"], + "unbound_ready_bytes": item["unbound_ready_bytes"], + "stale_count": item["stale_count"], + "stale_bytes": item["stale_bytes"], + } == { + "unbound_ready_count": 0, + "unbound_ready_bytes": 0, + "stale_count": 0, + "stale_bytes": 0, + } assert "producer_ref" not in usage.text usage_types = [usage.json()["items"][0]["scope_type"]] usage_cursor = usage.json()["next_cursor"] diff --git a/backend/tests/test_default_pre_submit_execution.py b/backend/tests/test_default_pre_submit_execution.py index 2531dbd52..3368a8c9d 100644 --- a/backend/tests/test_default_pre_submit_execution.py +++ b/backend/tests/test_default_pre_submit_execution.py @@ -29,7 +29,15 @@ ArtifactScratchManager, ) from app.modules.artifacts.schemas import ArtifactAuthorityDeniedError -from app.modules.artifacts.models import SubmissionBundleDurableIntent +from app.modules.artifacts.models import ( + ArtifactContent, + ArtifactOperationReceipt, + ArtifactPutAttempt, + ArtifactReplica, + ArtifactVerificationJob, + ArtifactVerificationReceipt, + SubmissionBundleDurableIntent, +) from app.modules.artifacts.service import ( ArtifactAdmissionRelationshipError, ArtifactAdmissionService, @@ -39,6 +47,12 @@ SubmissionBundleDurablePutRequest, SubmissionBundleDurablePutService, ) +from app.modules.artifacts.submission_admission_publication import ( + SubmissionBundleAdmissionPublisher, +) +from app.modules.artifacts.operator import ArtifactOperatorService +from app.modules.artifacts.metrics import artifact_admission_metrics +from app.modules.artifacts.schemas import ArtifactOperatorAuthorizationEvidence from app.modules.artifacts.submission_authorization import ( DenySubmissionBundlePreparedAuthorization, ) @@ -56,6 +70,7 @@ PreparedBundleMaterializationService, ) from app.modules.authorization.prepared import PreparedAuthorizationHandle +from app.modules.authorization.catalogue import ACTION_BY_ID from app.modules.checkers.catalogue import ( PreSubmissionCheckerPhase, build_pre_submission_checker_catalogue, @@ -89,6 +104,15 @@ async def consume(self, *, prepared_authorization, facts) -> None: self.facts = facts +class _AllowOperatorAuthority: + async def authorize(self, *, facts, **_values) -> ArtifactOperatorAuthorizationEvidence: + return ArtifactOperatorAuthorizationEvidence( + action_id=facts.action_id, + permission_id=ACTION_BY_ID[facts.action_id].permission_id.value, + decision_id=uuid4(), + ) + + @pytest.mark.asyncio async def test_evidence_workflow_requires_transaction_free_session() -> None: materialization = SimpleNamespace(materialize_prepared_bundle=lambda _request: None) @@ -438,6 +462,7 @@ async def test_effective_evidence_workflow_persists_once_and_replays_exactly( "artifact_contents", "artifact_replicas", "artifact_put_attempts", + "submission_bundle_admissions", "submissions", "checker_runs", "review_queue_entries", @@ -799,6 +824,110 @@ async def fresh_checked_bundle(): pass_capability=denied.pass_capability, ) ) + async with session.begin(): + attempt = await session.get( + ArtifactPutAttempt, + str(first_admission.attempt_id), + with_for_update=True, + ) + assert attempt is not None + content = ArtifactContent( + id=str(uuid4()), + sha256=attempt.sha256, + byte_count=attempt.byte_count, + media_type=attempt.media_type, + normalized_display_name=None, + ) + replica = ArtifactReplica( + id=str(uuid4()), + content_id=content.id, + storage_namespace_id=attempt.storage_namespace_id, + namespace_fingerprint=attempt.namespace_fingerprint, + adapter="local", + provider_profile="test", + provider_object_ref=attempt.canonical_target, + verification_state="verified", + availability_state="available", + integrity_state="valid", + ) + session.add_all((content, replica)) + await session.flush() + put_receipt = ArtifactOperationReceipt( + id=str(uuid4()), + put_attempt_id=attempt.id, + guide_source_item_id=None, + checker_run_id=None, + logical_role=None, + replica_id=replica.id, + operation="put", + idempotency_key=attempt.operation_identity, + request_digest=attempt.request_digest, + provider_object_ref=attempt.canonical_target, + replayed=False, + outcome="stored_pending_verification", + attempt_number=1, + correlation_id=attempt.operation_identity, + details=[], + ) + job = ArtifactVerificationJob( + id=str(uuid4()), + originating_put_attempt_id=attempt.id, + replica_id=replica.id, + status="verified", + maximum_attempts=5, + execution_generation=1, + ) + session.add_all((put_receipt, job)) + attempt.status = "object_confirmed" + attempt.replica_id = replica.id + attempt.receipt_id = put_receipt.id + verification_receipt = ArtifactVerificationReceipt( + id=str(uuid4()), + verification_job_id=job.id, + execution_generation=1, + outcome="verified", + observed_sha256=attempt.sha256, + observed_byte_count=attempt.byte_count, + ) + session.add(verification_receipt) + await session.flush() + job_id = job.id + verification_receipt_id = verification_receipt.id + attempt_byte_count = attempt.byte_count + + async def publish_ready() -> str: + async with session_factory() as publisher_session: + async with publisher_session.begin(): + ready = await SubmissionBundleAdmissionPublisher( + publisher_session + ).publish_verified( + verification_job_id=job_id, + verification_receipt_id=verification_receipt_id, + ) + assert ready is not None and ready.status == "ready" + return ready.id + + published_ids = await asyncio.gather(publish_ready(), publish_ready()) + assert published_ids[0] == published_ids[1] + async with session.begin(): + usage = await ArtifactOperatorService( + session, + _AllowOperatorAuthority(), + admission_settings, + artifact_admission_metrics, + ).admission_usage( + authorization_context=cast(Any, object()), + project_id=lineage.project_id, + task_id=request.task_id, + cursor=None, + limit=10, + ) + assert len(usage.items) == 3 + assert all(item["unbound_ready_count"] == 1 for item in usage.items) + assert all( + item["unbound_ready_bytes"] == attempt_byte_count for item in usage.items + ) + assert all(item["stale_count"] == 0 for item in usage.items) durable_counts = { table: int(await session.scalar(text(f"select count(*) from {table}")) or 0) for table in ( @@ -828,6 +957,47 @@ async def fresh_checked_bundle(): provider.execute_committed_put.assert_not_awaited() provider.resume_committed_put.assert_not_awaited() assert selected_evidence_id == first.evidence.evidence_set_id + await session.execute( + text( + "update submission_bundle_admissions set status='stale', " + "stale_at=now(), stale_reason='predecessor_advanced' where id=:id" + ), + {"id": published_ids[0]}, + ) + await session.commit() + async with session.begin(): + stale_usage = await ArtifactOperatorService( + session, + _AllowOperatorAuthority(), + admission_settings, + artifact_admission_metrics, + ).admission_usage( + authorization_context=cast(Any, object()), + project_id=lineage.project_id, + task_id=request.task_id, + cursor=None, + limit=10, + ) + assert all(item["unbound_ready_count"] == 0 for item in stale_usage.items) + assert all(item["stale_count"] == 1 for item in stale_usage.items) + assert all(item["stale_bytes"] == attempt_byte_count for item in stale_usage.items) + with pytest.raises(DBAPIError): + await session.execute( + text( + "update submission_bundle_admissions set archive_sha256=:digest " + "where id=:id" + ), + {"id": published_ids[0], "digest": "sha256:" + "9" * 64}, + ) + await session.commit() + await session.rollback() + with pytest.raises(DBAPIError): + await session.execute( + text("delete from submission_bundle_admissions where id=:id"), + {"id": published_ids[0]}, + ) + await session.commit() + await session.rollback() blocked_prepared = await preparation.prepare( _bytes(_archive("task.toml")), media_type="application/zip" ) @@ -958,7 +1128,10 @@ async def fresh_checked_bundle(): assert result_count == 5 * len(request.effective_plan.entries) assert after == { **before, + "artifact_contents": before["artifact_contents"] + 1, + "artifact_replicas": before["artifact_replicas"] + 1, "artifact_put_attempts": before["artifact_put_attempts"] + 1, + "submission_bundle_admissions": before["submission_bundle_admissions"] + 1, } diff --git a/backend/tests/test_submission_bundle_admission.py b/backend/tests/test_submission_bundle_admission.py index 3c4fa4bfb..158f0d9c1 100644 --- a/backend/tests/test_submission_bundle_admission.py +++ b/backend/tests/test_submission_bundle_admission.py @@ -3,8 +3,10 @@ from __future__ import annotations import asyncio +from contextlib import asynccontextmanager +from datetime import UTC, datetime from types import SimpleNamespace -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, Mock from uuid import uuid4 import pytest @@ -22,21 +24,59 @@ ) from app.modules.artifacts.submission_authorization import ( DenySubmissionBundlePreparedAuthorization, + DenySubmissionBundlePreparationAuthorization, ) from app.modules.artifacts.submission_admission import ( SubmissionBundleDurablePutRequest, SubmissionBundleDurablePutService, ) +from app.modules.artifacts.submission_admission_publication import ( + SubmissionBundleAdmissionPublicationError, + SubmissionBundleAdmissionPublisher, +) from app.modules.artifacts.service import ArtifactAdmissionService from app.modules.artifacts.submission_custody import SubmissionBundlePreparedCustody +from app.modules.artifacts.submission_preparation import ( + PreparedSubmissionBundlePreparationCommand, + SubmissionBundlePreparationRejected, +) +from app.interfaces.artifact_operations import SubmissionBundlePreparationRequest +from app.modules.authorization.runtime import ( + ActorKind, + ActorStatus, + HumanAuthorizationContext, + IdentityLinkStatus, +) from app.modules.authorization.prepared import PreparedAuthorizationHandle from tests.artifact_store_helpers import artifact_byte_stream, artifact_preparation_limits +from app.main import create_app +from app.modules.tasks.router import router as tasks_router def _sha(character: str) -> str: return "sha256:" + character * 64 +def test_submission_bundle_preparation_route_is_hidden() -> None: + app = create_app() + route = next( + route + for route in tasks_router.routes + if getattr(route, "name", None) == "prepare_submission_bundle" + ) + assert route.include_in_schema is False + mounted = app.url_path_for("prepare_submission_bundle", task_id=str(uuid4())) + assert str(mounted).startswith("/api/v1/tasks/") + assert str(mounted).endswith("/submission-bundle-preparations") + assert route.methods == {"POST"} + assert "/api/v1/tasks/{task_id}/submission-bundle-preparations" not in app.openapi()["paths"] + + +@asynccontextmanager +async def _transaction(): + yield + + async def _prepared(tmp_path): manager = ArtifactScratchManager( root=tmp_path / "scratch", @@ -49,6 +89,87 @@ async def _prepared(tmp_path): return manager, prepared +@pytest.mark.asyncio +async def test_hidden_preparation_denies_before_reading_uploaded_bytes() -> None: + reads = 0 + + async def bytes_source(): + nonlocal reads + reads += 1 + yield b"PK\x03\x04must-not-be-read" + + def runtime_factory(): + raise AssertionError("runtime must not open before contributor preflight") + + command = PreparedSubmissionBundlePreparationCommand( + session=SimpleNamespace(), + authority=DenySubmissionBundlePreparationAuthorization(), + runtime_factory=runtime_factory, + ) + with pytest.raises(ArtifactAuthorityDeniedError): + await command.prepare( + SubmissionBundlePreparationRequest( + authorization_context=HumanAuthorizationContext( + actor_profile_id=uuid4(), + actor_kind=ActorKind.HUMAN, + actor_status=ActorStatus.ACTIVE, + identity_link_id=uuid4(), + identity_link_status=IdentityLinkStatus.ACTIVE, + request_id=uuid4(), + correlation_id=uuid4(), + ), + task_id=uuid4(), + assignment_id=uuid4(), + predecessor_submission_id=None, + idempotency_key=uuid4(), + summary="summary", + contributor_attestation="attestation", + media_type="application/zip", + byte_source=bytes_source(), + ) + ) + assert reads == 0 + + +@pytest.mark.asyncio +async def test_hidden_preparation_closes_authority_after_invalid_media_type() -> None: + authority = SimpleNamespace( + preflight=AsyncMock(), + close=Mock(), + ) + command = PreparedSubmissionBundlePreparationCommand( + session=SimpleNamespace(), + authority=authority, + runtime_factory=Mock(side_effect=AssertionError("runtime must stay closed")), + ) + with pytest.raises( + SubmissionBundlePreparationRejected, + match="submission_bundle_media_type_invalid", + ): + await command.prepare( + SubmissionBundlePreparationRequest( + authorization_context=HumanAuthorizationContext( + actor_profile_id=uuid4(), + actor_kind=ActorKind.HUMAN, + actor_status=ActorStatus.ACTIVE, + identity_link_id=uuid4(), + identity_link_status=IdentityLinkStatus.ACTIVE, + request_id=uuid4(), + correlation_id=uuid4(), + ), + task_id=uuid4(), + assignment_id=uuid4(), + predecessor_submission_id=None, + idempotency_key=uuid4(), + summary="summary", + contributor_attestation="attestation", + media_type="application/json", + byte_source=artifact_byte_stream(b"{}"), + ) + ) + authority.close.assert_called_once_with() + + def _capability(prepared, evidence_set_id): service = PreSubmitEvidenceService(SimpleNamespace()) return service._mint_pass_capability( @@ -62,6 +183,225 @@ def _capability(prepared, evidence_set_id): ) +@pytest.mark.asyncio +@pytest.mark.parametrize("observed_confirmation", [False, True]) +async def test_verified_submission_lineage_publishes_one_ready_admission( + observed_confirmation: bool, +) -> None: + attempt_id = str(uuid4()) + replica_id = str(uuid4()) + content_id = str(uuid4()) + job_id = str(uuid4()) + receipt_id = str(uuid4()) + evidence_id = str(uuid4()) + digest = _sha("1") + evidence = SimpleNamespace( + id=evidence_id, + terminal_status="passed", + eligible=True, + archive_sha256=digest, + archive_byte_count=25, + actor_profile_id=str(uuid4()), + identity_link_id=str(uuid4()), + project_id=str(uuid4()), + task_id=str(uuid4()), + assignment_id=str(uuid4()), + predecessor_submission_id=None, + predecessor_submission_version=None, + semantic_manifest_id=str(uuid4()), + semantic_manifest_sha256=_sha("2"), + guide_id=str(uuid4()), + guide_version="1", + source_snapshot_id=str(uuid4()), + source_snapshot_sha256=_sha("3"), + locked_guide_sha256=_sha("4"), + effective_policy_id=str(uuid4()), + locked_artifact_policy_sha256=_sha("5"), + pre_submit_policy_id=str(uuid4()), + locked_checker_policy_sha256=_sha("6"), + effective_plan_sha256=_sha("7"), + locked_policy_context_hash=_sha("8"), + ) + attempt = SimpleNamespace( + id=attempt_id, + producer_request_type="submission_bundle", + producer_ref=evidence.actor_profile_id, + project_id=evidence.project_id, + task_id=evidence.task_id, + sha256=digest, + byte_count=25, + media_type="application/zip", + replica_id=replica_id, + receipt_id=None if observed_confirmation else str(uuid4()), + ) + job = SimpleNamespace( + id=job_id, + originating_put_attempt_id=attempt_id, + replica_id=replica_id, + execution_generation=2, + ) + intent = SimpleNamespace(id=str(uuid4()), pre_submit_evidence_set_id=evidence_id) + replica = SimpleNamespace( + id=replica_id, + content_id=content_id, + verification_state="verified", + availability_state="available", + integrity_state="valid", + ) + content = SimpleNamespace(id=content_id, sha256=digest, byte_count=25) + verification = SimpleNamespace( + id=receipt_id, + verification_job_id=job_id, + execution_generation=2, + outcome="verified", + observed_sha256=digest, + observed_byte_count=25, + ) + put_receipt = SimpleNamespace(id=str(uuid4()) if observed_confirmation else attempt.receipt_id) + now = datetime.now(UTC) + session = SimpleNamespace( + scalar=AsyncMock( + side_effect=[ + job, + attempt, + intent, + None, + evidence, + replica, + verification, + content, + put_receipt, + now, + ] + ), + add=Mock(), + flush=AsyncMock(), + ) + + admission = await SubmissionBundleAdmissionPublisher(session).publish_verified( + verification_job_id=job_id, + verification_receipt_id=receipt_id, + ) + + assert admission is not None + assert admission.status == "ready" + assert admission.pre_submit_evidence_set_id == evidence_id + assert admission.artifact_content_id == content_id + assert admission.put_operation_receipt_id == ( + None if observed_confirmation else attempt.receipt_id + ) + assert admission.put_observation_receipt_id == ( + put_receipt.id if observed_confirmation else None + ) + session.add.assert_called_once_with(admission) + + +@pytest.mark.asyncio +async def test_observed_confirmed_write_receipt_is_supported() -> None: + attempt = SimpleNamespace( + id=str(uuid4()), + receipt_id=None, + sha256=_sha("1"), + byte_count=25, + ) + observation = SimpleNamespace(id=str(uuid4())) + session = SimpleNamespace(scalar=AsyncMock(return_value=observation)) + + operation_id, observation_id = await SubmissionBundleAdmissionPublisher( + session + )._write_receipt_ids(attempt) + + assert operation_id is None + assert observation_id == observation.id + + +@pytest.mark.asyncio +async def test_verified_guide_content_does_not_publish_submission_admission() -> None: + job = SimpleNamespace(id=str(uuid4()), originating_put_attempt_id=str(uuid4())) + attempt = SimpleNamespace(id=job.originating_put_attempt_id, producer_request_type="guide") + session = SimpleNamespace( + scalar=AsyncMock(side_effect=[job, attempt]), + add=Mock(), + ) + + result = await SubmissionBundleAdmissionPublisher(session).publish_verified( + verification_job_id=job.id, + verification_receipt_id=str(uuid4()), + ) + + assert result is None + session.add.assert_not_called() + + +@pytest.mark.asyncio +async def test_unverified_or_mismatched_lineage_is_not_publishable() -> None: + digest = _sha("1") + evidence = SimpleNamespace( + id=str(uuid4()), + terminal_status="failed", + eligible=False, + archive_sha256=digest, + archive_byte_count=25, + actor_profile_id=str(uuid4()), + project_id=str(uuid4()), + task_id=str(uuid4()), + ) + attempt = SimpleNamespace( + id=str(uuid4()), + sha256=digest, + byte_count=25, + producer_ref=evidence.actor_profile_id, + project_id=evidence.project_id, + task_id=evidence.task_id, + media_type="application/zip", + replica_id=str(uuid4()), + ) + job = SimpleNamespace( + id=str(uuid4()), + originating_put_attempt_id=attempt.id, + replica_id=attempt.replica_id, + execution_generation=1, + ) + replica = SimpleNamespace( + id=attempt.replica_id, + content_id=str(uuid4()), + verification_state="verified", + availability_state="available", + integrity_state="valid", + ) + content = SimpleNamespace(id=replica.content_id, sha256=digest, byte_count=25) + receipt = SimpleNamespace( + verification_job_id=job.id, + execution_generation=1, + outcome="integrity_mismatch", + observed_sha256=digest, + observed_byte_count=25, + ) + + assert not SubmissionBundleAdmissionPublisher._matches_verified_lineage( + evidence, attempt, job, replica, content, receipt + ) + intent = SimpleNamespace(id=str(uuid4()), pre_submit_evidence_set_id=evidence.id) + session = SimpleNamespace( + scalar=AsyncMock( + side_effect=[job, attempt, intent, None, evidence, replica, receipt, content] + ), + add=Mock(), + flush=AsyncMock(), + ) + attempt.producer_request_type = "submission_bundle" + with pytest.raises( + SubmissionBundleAdmissionPublicationError, + match="verified submission bundle lineage does not match", + ): + await SubmissionBundleAdmissionPublisher(session).publish_verified( + verification_job_id=job.id, + verification_receipt_id=str(uuid4()), + ) + session.add.assert_not_called() + session.flush.assert_not_awaited() + + @pytest.mark.asyncio async def test_durable_put_admits_in_transaction_then_publishes(tmp_path) -> None: manager, prepared = await _prepared(tmp_path) @@ -72,6 +412,8 @@ async def test_durable_put_admits_in_transaction_then_publishes(tmp_path) -> Non sync_session=SimpleNamespace(get_transaction=lambda: transaction), in_nested_transaction=lambda: False, in_transaction=lambda: state["transaction"], + scalar=AsyncMock(return_value=None), + begin=_transaction, ) admission_result = ArtifactAdmissionResult( attempt_id=uuid4(), @@ -278,6 +620,8 @@ async def admit(request, **values): sync_session=SimpleNamespace(get_transaction=lambda: SimpleNamespace(is_active=True)), in_nested_transaction=lambda: False, in_transaction=lambda: state["transaction"], + scalar=AsyncMock(return_value=None), + begin=_transaction, ), admission=SimpleNamespace(admit=admit), storage=storage, @@ -306,6 +650,44 @@ async def admit(request, **values): manager.close() +@pytest.mark.asyncio +async def test_object_confirmed_replay_does_not_reclaim_provider_work(tmp_path) -> None: + manager, prepared = await _prepared(tmp_path) + storage = SimpleNamespace( + execute_committed_put=AsyncMock(), + resume_committed_put=AsyncMock(), + ) + service = SubmissionBundleDurablePutService( + session=SimpleNamespace( + in_transaction=lambda: False, + scalar=AsyncMock(return_value=None), + begin=_transaction, + ), + admission=SimpleNamespace(), + storage=storage, + authorization=object(), + ) + try: + result = await service.publish_after_commit( + prepared, + uuid4(), + ArtifactAdmissionResult( + attempt_id=uuid4(), + status="object_confirmed", + operation_identity=_sha("3"), + request_digest=_sha("4"), + charge_ids=(uuid4(),), + replayed=True, + ), + ) + assert result.status == "object_confirmed" + storage.execute_committed_put.assert_not_awaited() + storage.resume_committed_put.assert_not_awaited() + finally: + await prepared.close() + manager.close() + + @pytest.mark.asyncio async def test_durable_put_rejects_capability_replay_before_admission(tmp_path) -> None: manager, prepared = await _prepared(tmp_path) diff --git a/docs/architecture_data_model.md b/docs/architecture_data_model.md index 9b6e9d612..c3d6cccc0 100644 --- a/docs/architecture_data_model.md +++ b/docs/architecture_data_model.md @@ -824,6 +824,15 @@ attempt, authorization evidence, and this join commit before provider I/O. Generic observation and recovery continue from the put attempt after process loss. +ART-04C2 adds one `SubmissionBundleAdmission` only when the verifier's complete +read produces a matching successful receipt. The row joins the exact durable +intent and evidence to provider-neutral content, the verified replica, the +verification receipt, and exactly one direct-put or observed-confirmed receipt. +It starts in `ready`; later 05A alone may execute `ready -> consumed|stale`. +The hidden preparation route keeps scratch and prepared handles process-local, +returns before durable verification finishes, and remains excluded from OpenAPI +and unavailable until AUTH activates `artifact.submission_bundle.prepare`. + Blocking pre-submit failures prevent submission creation, create no submission row, no submission version, no task transition to `submitted`, and no submission-created audit event. Workstream still writes a task audit event named diff --git a/docs/operations_artifact_storage.md b/docs/operations_artifact_storage.md index 7c8c4b668..f5ab8085d 100644 --- a/docs/operations_artifact_storage.md +++ b/docs/operations_artifact_storage.md @@ -47,6 +47,14 @@ HTTP response and audit identifiers. It never enumerates producer-scope identifiers. It is read-only: it cannot release charges, change configuration, or create recovery work. +Each bounded scope also reports submission-bundle pressure as +`unbound_ready_count`/`unbound_ready_bytes` and +`stale_count`/`stale_bytes`. Ready admissions are verified but not yet consumed +by a Submission; stale admissions proved incompatible at a later consumption +boundary. Both remain charged under v0.1 capacity limits. Use these projections +to distinguish abandoned or context-invalid bundles from ordinary artifact +growth; there is no operator deletion or release action. + Every successful admission transaction emits `workstream_artifact_admission_pressure_total` for each derived deployment, project, producer, and task scope. The bounded @@ -64,6 +72,11 @@ expected, confirm that its parent scopes have capacity, and compare the configured limit with the persisted scope limit. Do not delete admission charges or edit counters in PostgreSQL. +Migration `0061_submission_admission` installs immutable ready-admission +custody. Downgrade is intentionally refused once admission rows exist; rollback +planning must preserve those facts and move forward with a corrective migration +rather than deleting or bypassing them. + ## Quota expansion and rollback Quota changes are configuration-driven. Increase the smallest affected scope diff --git a/docs/spec_artifact_storage_service.md b/docs/spec_artifact_storage_service.md index c094885c3..c8ef18431 100644 --- a/docs/spec_artifact_storage_service.md +++ b/docs/spec_artifact_storage_service.md @@ -363,12 +363,13 @@ get_verification_job/get_recovery_attempt/list_audit_events/admission_usage(...) ArtifactOperatorRecoveryPort.retry_verification(ArtifactRecoveryRequest) ``` -Every durable mutation request in this port family except the unchanged -Operator recovery request is a process-local, -non-Pydantic value carrying one opaque `PreparedAuthorizationHandle`. The -typed method fixes the expected action; requests contain no caller-selected -ActionId, generic resource context, or facts map. Handles never enter route -schemas, outbox/Celery payloads, provider interfaces, or serialized contracts. +Every protected durable mutation in this port family consumes one opaque, +process-local `PreparedAuthorizationHandle` at its transaction boundary. The +hidden HTTP submission-bundle preparation request carries authenticated request +context rather than a prepared handle; ART obtains separate prepared handles +immediately before materialization and durable put intent. The typed methods fix +their expected actions, and handles never enter route schemas, outbox/Celery +payloads, provider interfaces, or serialized contracts. `GuideArtifactIngestRequest` contains prepared authority, exact project, guide, guide-source snapshot and item IDs, logical role, media type, and authorized @@ -377,9 +378,9 @@ reference. `GuideSourceMaterializationRequest` contains an idempotency key and the exact guide, source, setup-generation, and binding identifiers; it contains no prepared handle. The materializer creates and consumes fresh fixed-reader authority inside the same root transaction that locks the canonical read facts. -`SubmissionBundlePreparationRequest` contains -prepared authority, contributor task/assignment selectors, and one outer ZIP -byte source. There is no upload-session compatibility port. +`SubmissionBundlePreparationRequest` contains authenticated request context, +contributor task/assignment selectors, and one outer ZIP byte source. There is +no upload-session compatibility port. `PreparedBundleMaterializationRequest` is internal and process-local; it wraps only prepared authority, the exact task/assignment context, the current `PreparedArtifact` generation, and exact policy/checker selectors. @@ -1269,15 +1270,21 @@ One immutable verified preparation with closed status: ```text status = ready | consumed | stale +durable_intent_id = UUID +put_attempt_id = UUID actor_profile_id = UUID -preparation_identity_link_id = UUID +identity_link_id = UUID project_id = UUID task_id = UUID assignment_id = UUID predecessor_submission_id = UUID | null -locked_task_context_id/hash = immutable canonical context reference +locked_policy_context_hash = SHA-256 copied from immutable pre-submit evidence artifact_content_id = UUID -submission_bundle_manifest_id = UUID +verified_replica_id = UUID +verification_receipt_id = UUID +put_operation_receipt_id = UUID | null +put_observation_receipt_id = UUID | null +semantic_manifest_id/hash = immutable canonical manifest identity pre_submit_evidence_set_id = UUID ready_at = timestamp consumed_at = timestamp | null @@ -1291,9 +1298,21 @@ task, assignment, immediate predecessor, exact locked task/guide/policy context, `ArtifactContent`, `SubmissionBundleManifest`, immutable pre-submit evidence-set identity, timestamps, terminal reason, and optional consuming Submission. Status-field constraints require terminal facts only in their matching state. +A matching hash is first persisted on the immutable evidence set and the +admission trigger requires the copied value, so a direct insert cannot invent a +different policy-context identity. A unique Submission admission reference plus locked/CAS transition permits at most one consumer. +ART-04C2 publishes `ready` only inside the generic verifier's successful +terminal transaction through a narrow submission publisher. It reloads and +matches the durable intent, passing evidence, put attempt, content, verified +replica, and successful full-read verification receipt. Direct provider +acknowledgement and observed-confirmed recovery use distinct nullable receipt +references; exactly one must match. Guide and checker-output verification do +not create submission admissions. The hidden preparation POST returns bounded +operation/current-admission state and does not wait for verification. + `ready -> consumed` occurs only in the transaction creating Submission and binding. `ready -> stale` occurs only when consumption proves task closure, predecessor advancement, or locked-context replacement. Authority loss alone From 271749d659a46ab3c0c06beff3d12f72edc4e66f Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 8 Aug 2026 17:43:53 +0100 Subject: [PATCH 2/7] Address ART-04C2 review findings --- ...S-ART-001-04C2-external-review-response.md | 47 +++++++++++++ .../0061_submission_bundle_admission.py | 11 +++- backend/app/modules/artifacts/models.py | 9 ++- backend/app/modules/artifacts/service.py | 18 +++-- backend/app/modules/tasks/router.py | 21 ++++++ backend/tests/conftest.py | 2 +- backend/tests/test_alembic.py | 66 ++++++++++++++++++- .../test_effective_pre_submit_execution.py | 12 +++- .../tests/test_submission_bundle_admission.py | 14 +++- 9 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 .agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-04C2-external-review-response.md diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-04C2-external-review-response.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-04C2-external-review-response.md new file mode 100644 index 000000000..df6a6efe4 --- /dev/null +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-04C2-external-review-response.md @@ -0,0 +1,47 @@ +# WS-ART-001-04C2 External Review Response + +## Comments addressed + +- Verifier publication failure now converts the same fenced verification + receipt and job to terminal `conflict`, preserving bounded attempt accounting + without appending a contradictory second receipt. +- A partial unique database index now guarantees that one Submission ID cannot + consume multiple admissions. +- Summary and contributor-attestation header values must be ASCII before they + enter immutable packet evidence; invalid encoding returns bounded `422`. +- Locked task/predecessor/policy races now return bounded context-changed `409` + rather than an internal error. +- The evidence policy-context constraint now uses the repository's logical + naming-convention suffix. +- Alembic round-trip inspection now covers the evidence column/constraint, + restored immutability trigger, and consumer uniqueness index. + +## Comments deferred or declined + +- The migration intentionally retains its local canonical-JSON implementation + instead of importing mutable application code. Its encoding exactly matches + `canonical_json_hash`, while keeping historical migration execution + self-contained. +- CodeRabbit's generic docstring-percentage warning is not treated as a code + failure: the repository's authoritative Agent Gates passed, and new boundary + helpers have focused docstrings. +- Predicate-by-predicate negative publisher tests remain a low-risk follow-up; + the current service-negative, database-trigger, and real concurrent workflow + tests cover the complete fail-closed boundary. + +## Human decisions needed + +None. + +## Commands rerun + +- Focused Ruff and formatting for every changed response file. +- `git diff --check`. +- PostgreSQL migration upgrade from `0059` through `0061`. +- Canonical schema fingerprint validation after the new constraint and index. +- Focused admission tests and all hosted Backend lanes on the updated PR head. + +## Remaining risks + +AUTH activation and admission consumption remain separate reviewed chunks. The +route remains hidden and unavailable in this PR. diff --git a/backend/alembic/versions/0061_submission_bundle_admission.py b/backend/alembic/versions/0061_submission_bundle_admission.py index a5a1d820f..c9b033848 100644 --- a/backend/alembic/versions/0061_submission_bundle_admission.py +++ b/backend/alembic/versions/0061_submission_bundle_admission.py @@ -59,7 +59,7 @@ def upgrade() -> None: ) op.alter_column("pre_submit_evidence_sets", "locked_policy_context_hash", nullable=False) op.create_check_constraint( - "ck_pre_submit_evidence_policy_context_sha256", + "policy_context_sha256", "pre_submit_evidence_sets", "locked_policy_context_hash ~ '^sha256:[0-9a-f]{64}$'", ) @@ -176,6 +176,13 @@ def upgrade() -> None: op.create_index( f"ix_submission_bundle_admissions_{column}", "submission_bundle_admissions", [column] ) + op.create_index( + "uq_submission_bundle_admission_consumer", + "submission_bundle_admissions", + ["consumed_by_submission_id"], + unique=True, + postgresql_where=sa.text("consumed_by_submission_id is not null"), + ) op.execute( """ create function guard_submission_bundle_admission_verified_lineage() @@ -314,7 +321,7 @@ def downgrade() -> None: op.execute("drop function guard_submission_bundle_admission_verified_lineage()") op.drop_table("submission_bundle_admissions") op.drop_constraint( - "ck_pre_submit_evidence_policy_context_sha256", + "policy_context_sha256", "pre_submit_evidence_sets", type_="check", ) diff --git a/backend/app/modules/artifacts/models.py b/backend/app/modules/artifacts/models.py index d78c7388a..ec047f8c1 100644 --- a/backend/app/modules/artifacts/models.py +++ b/backend/app/modules/artifacts/models.py @@ -17,6 +17,7 @@ String, Text, UniqueConstraint, + text, ) from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql import func @@ -150,7 +151,7 @@ class PreSubmitEvidenceSet(Base): ), CheckConstraint( SHA256_CHECK.format(column="locked_policy_context_hash"), - name="ck_pre_submit_evidence_policy_context_sha256", + name="policy_context_sha256", ), CheckConstraint( SHA256_CHECK.format(column="result_manifest_sha256"), @@ -1081,6 +1082,12 @@ class SubmissionBundleAdmission(Base): UniqueConstraint( "verification_receipt_id", name="uq_submission_bundle_admission_verification" ), + Index( + "uq_submission_bundle_admission_consumer", + "consumed_by_submission_id", + unique=True, + postgresql_where=text("consumed_by_submission_id is not null"), + ), CheckConstraint("status in ('ready','consumed','stale')", name="status"), CheckConstraint( SHA256_CHECK.format(column="locked_policy_context_hash"), diff --git a/backend/app/modules/artifacts/service.py b/backend/app/modules/artifacts/service.py index 820100ff6..0a89aee61 100644 --- a/backend/app/modules/artifacts/service.py +++ b/backend/app/modules/artifacts/service.py @@ -1276,13 +1276,23 @@ async def _complete_verification( ) if outcome == "verified": from app.modules.artifacts.submission_admission_publication import ( + SubmissionBundleAdmissionPublicationError, SubmissionBundleAdmissionPublisher, ) - await SubmissionBundleAdmissionPublisher(self._session).publish_verified( - verification_job_id=job.id, - verification_receipt_id=verification_receipt.id, - ) + try: + await SubmissionBundleAdmissionPublisher(self._session).publish_verified( + verification_job_id=job.id, + verification_receipt_id=verification_receipt.id, + ) + except SubmissionBundleAdmissionPublicationError: + # Valid bytes can still have permanently inadmissible + # submission lineage. Keep one fenced terminal receipt and + # prevent an endless verifier retry loop. + outcome = "conflict" + verification_receipt.outcome = "conflict" + verification_receipt.observed_sha256 = None + verification_receipt.observed_byte_count = None job.status = outcome job.next_run_at = None job.terminal_result_code = outcome diff --git a/backend/app/modules/tasks/router.py b/backend/app/modules/tasks/router.py index fb3ebe710..5ad19f1ae 100644 --- a/backend/app/modules/tasks/router.py +++ b/backend/app/modules/tasks/router.py @@ -40,6 +40,7 @@ TaskWorkContextResponse, TaskWithAssignmentResponse, ) +from app.modules.tasks.pre_submit_context import PreSubmitLockedContextInvalid from app.modules.tasks.service import TaskService, TaskServiceError from app.schemas.auth import ActorContext @@ -76,6 +77,20 @@ class SubmissionBundlePreparationResponse(BaseModel): {"$ref": "#/components/schemas/HTTPValidationError"}, ] } + + +def _require_ascii_submission_packet_headers(summary: str, attestation: str) -> None: + """Reject lossy HTTP-header decoding before immutable evidence hashing.""" + try: + summary.encode("ascii") + attestation.encode("ascii") + except UnicodeEncodeError as exc: + raise HTTPException( + status_code=422, + detail="submission_bundle_packet_header_encoding_invalid", + ) from exc + + TASK_LOCKED_CONTEXT_DOMAIN_ERROR_RESPONSE_SCHEMA = { "oneOf": [ { @@ -420,6 +435,7 @@ async def prepare_submission_bundle( raise HTTPException(status_code=404, detail="Task not found") assert assignment_id is not None and idempotency_key is not None assert summary is not None and contributor_attestation is not None + _require_ascii_submission_packet_headers(summary, contributor_attestation) try: identifiers = ( UUID(task_id), @@ -447,6 +463,11 @@ async def prepare_submission_bundle( raise HTTPException(status_code=404, detail="Task not found") from exc except SubmissionBundlePreparationRejected as exc: raise HTTPException(status_code=422, detail=str(exc)) from exc + except PreSubmitLockedContextInvalid as exc: + raise HTTPException( + status_code=409, + detail="submission_bundle_preparation_context_changed", + ) from exc return SubmissionBundlePreparationResponse.model_validate(result, from_attributes=True) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 672d3ef6e..080fce28a 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -21,7 +21,7 @@ from scripts.run_isolated_tests import LOOPBACK, NAME_RE, ROLE_RE DDL_LOCK_DIRECTORY = Path("/tmp") -EXPECTED_PUBLIC_SCHEMA_SHA256 = "c52b16256a213cfc03827fd8f8107420cf0571dbf8ca839c3d8ff1b1f564ed60" +EXPECTED_PUBLIC_SCHEMA_SHA256 = "f22478cb0230f14e7ffd7c4494e519b54daff046e19ca6dfde9fcf53dc99a144" PROTECTED_TEST_TABLES = ( "actor_profile_migration_state", "alembic_version", diff --git a/backend/tests/test_alembic.py b/backend/tests/test_alembic.py index a78b27e68..a51c4729a 100644 --- a/backend/tests/test_alembic.py +++ b/backend/tests/test_alembic.py @@ -321,7 +321,51 @@ async def _submission_bundle_admission_schema(database_url: str) -> dict[str, ob ) ).all() ) - return {"table_exists": table_exists, "constraints": constraints, "triggers": triggers} + evidence_column = bool( + await connection.scalar( + text( + "select count(*) from information_schema.columns " + "where table_schema='public' and table_name='pre_submit_evidence_sets' " + "and column_name='locked_policy_context_hash'" + ) + ) + ) + evidence_constraints = set( + ( + await connection.scalars( + text( + "select conname from pg_constraint where conrelid=" + "to_regclass('pre_submit_evidence_sets') and " + "conname='ck_pre_submit_evidence_sets_policy_context_sha256'" + ) + ) + ).all() + ) + evidence_immutable_enabled = await connection.scalar( + text( + "select tgenabled from pg_trigger where not tgisinternal and " + "tgrelid=to_regclass('pre_submit_evidence_sets') and " + "tgname='pre_submit_evidence_sets_immutable'" + ) + ) + consumer_index = bool( + await connection.scalar( + text( + "select count(*) from pg_indexes where schemaname='public' and " + "tablename='submission_bundle_admissions' and " + "indexname='uq_submission_bundle_admission_consumer'" + ) + ) + ) + return { + "table_exists": table_exists, + "constraints": constraints, + "triggers": triggers, + "evidence_column": evidence_column, + "evidence_constraints": evidence_constraints, + "evidence_immutable_enabled": evidence_immutable_enabled, + "consumer_index": consumer_index, + } finally: await engine.dispose() @@ -344,7 +388,19 @@ def test_0061_submission_bundle_admission_empty_round_trip( finally: command.upgrade(config, "head") - assert prior == restored == {"table_exists": False, "constraints": set(), "triggers": set()} + assert ( + prior + == restored + == { + "table_exists": False, + "constraints": set(), + "triggers": set(), + "evidence_column": False, + "evidence_constraints": set(), + "evidence_immutable_enabled": "O", + "consumer_index": False, + } + ) assert installed == repeated assert { "uq_submission_bundle_admission_intent", @@ -358,6 +414,12 @@ def test_0061_submission_bundle_admission_empty_round_trip( "submission_bundle_admission_lineage", "submission_bundle_admission_delete", } + assert installed["evidence_column"] is True + assert installed["evidence_constraints"] == { + "ck_pre_submit_evidence_sets_policy_context_sha256" + } + assert installed["evidence_immutable_enabled"] == "O" + assert installed["consumer_index"] is True async def _submission_policy_authority_shape(database_url: str) -> dict[str, object]: diff --git a/backend/tests/test_effective_pre_submit_execution.py b/backend/tests/test_effective_pre_submit_execution.py index 6ff13c33d..924955a29 100644 --- a/backend/tests/test_effective_pre_submit_execution.py +++ b/backend/tests/test_effective_pre_submit_execution.py @@ -283,8 +283,16 @@ async def test_locked_context_revalidates_identity_assignment_and_policy_lineage ), None, SimpleNamespace(id=str(guide_id), project_id=str(project_id), version="1"), - SimpleNamespace(id=str(effective_policy_id), effective_policy_hash=_sha("2")), - SimpleNamespace(id=str(checker_policy_id), compiled_bundle_hash=_sha("3")), + SimpleNamespace( + id=str(effective_policy_id), + effective_policy_hash=_sha("2"), + effective_policy={}, + ), + SimpleNamespace( + id=str(checker_policy_id), + compiled_bundle_hash=_sha("3"), + compiled_bundle={}, + ), ] ) ) diff --git a/backend/tests/test_submission_bundle_admission.py b/backend/tests/test_submission_bundle_admission.py index 158f0d9c1..cbdd5f033 100644 --- a/backend/tests/test_submission_bundle_admission.py +++ b/backend/tests/test_submission_bundle_admission.py @@ -10,6 +10,7 @@ from uuid import uuid4 import pytest +from fastapi import HTTPException from app.modules.artifacts.pre_submit_evidence import ( PreSubmitEvidenceConflict, @@ -50,7 +51,10 @@ from app.modules.authorization.prepared import PreparedAuthorizationHandle from tests.artifact_store_helpers import artifact_byte_stream, artifact_preparation_limits from app.main import create_app -from app.modules.tasks.router import router as tasks_router +from app.modules.tasks.router import ( + _require_ascii_submission_packet_headers, + router as tasks_router, +) def _sha(character: str) -> str: @@ -72,6 +76,14 @@ def test_submission_bundle_preparation_route_is_hidden() -> None: assert "/api/v1/tasks/{task_id}/submission-bundle-preparations" not in app.openapi()["paths"] +def test_submission_packet_headers_reject_non_ascii() -> None: + _require_ascii_submission_packet_headers("plain summary", "plain attestation") + with pytest.raises(HTTPException) as failure: + _require_ascii_submission_packet_headers("caf\N{LATIN SMALL LETTER E WITH ACUTE}", "ok") + assert failure.value.status_code == 422 + assert failure.value.detail == "submission_bundle_packet_header_encoding_invalid" + + @asynccontextmanager async def _transaction(): yield From f683433d576c4ca93e0bf0b62a5d0f19985ff1ff Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 8 Aug 2026 18:01:33 +0100 Subject: [PATCH 3/7] Match PostgreSQL trigger state type --- backend/tests/test_alembic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_alembic.py b/backend/tests/test_alembic.py index a51c4729a..70ae7fa9c 100644 --- a/backend/tests/test_alembic.py +++ b/backend/tests/test_alembic.py @@ -397,7 +397,7 @@ def test_0061_submission_bundle_admission_empty_round_trip( "triggers": set(), "evidence_column": False, "evidence_constraints": set(), - "evidence_immutable_enabled": "O", + "evidence_immutable_enabled": b"O", "consumer_index": False, } ) @@ -418,7 +418,7 @@ def test_0061_submission_bundle_admission_empty_round_trip( assert installed["evidence_constraints"] == { "ck_pre_submit_evidence_sets_policy_context_sha256" } - assert installed["evidence_immutable_enabled"] == "O" + assert installed["evidence_immutable_enabled"] == b"O" assert installed["consumer_index"] is True From b6e0e764fae1483deb13edf12d1a1b99b3d3058d Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sat, 8 Aug 2026 18:19:15 +0100 Subject: [PATCH 4/7] Cover admission replay projection --- .../tests/test_submission_bundle_admission.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/backend/tests/test_submission_bundle_admission.py b/backend/tests/test_submission_bundle_admission.py index cbdd5f033..49af275f8 100644 --- a/backend/tests/test_submission_bundle_admission.py +++ b/backend/tests/test_submission_bundle_admission.py @@ -29,6 +29,7 @@ ) from app.modules.artifacts.submission_admission import ( SubmissionBundleDurablePutRequest, + SubmissionBundleDurablePutResult, SubmissionBundleDurablePutService, ) from app.modules.artifacts.submission_admission_publication import ( @@ -40,6 +41,7 @@ from app.modules.artifacts.submission_preparation import ( PreparedSubmissionBundlePreparationCommand, SubmissionBundlePreparationRejected, + SubmissionBundlePreparationResult, ) from app.interfaces.artifact_operations import SubmissionBundlePreparationRequest from app.modules.authorization.runtime import ( @@ -182,6 +184,54 @@ async def test_hidden_preparation_closes_authority_after_invalid_media_type() -> authority.close.assert_called_once_with() +@pytest.mark.asyncio +async def test_existing_durable_preparation_projects_exact_ready_admission() -> None: + attempt_id = uuid4() + admission_id = uuid4() + row_result = SimpleNamespace( + one_or_none=lambda: ( + SimpleNamespace(id=str(uuid4())), + SimpleNamespace(id=str(attempt_id), status="object_confirmed"), + SimpleNamespace(id=str(admission_id)), + ) + ) + session = SimpleNamespace(begin=_transaction, execute=AsyncMock(return_value=row_result)) + command = PreparedSubmissionBundlePreparationCommand( + session=session, + authority=SimpleNamespace(), + runtime_factory=Mock(), + ) + + result = await command._existing_durable_result(uuid4()) + + assert result == SubmissionBundlePreparationResult( + put_attempt_id=attempt_id, + admission_id=admission_id, + status="ready", + replayed=True, + ) + + +def test_durable_put_result_projects_without_losing_replay_state() -> None: + durable = SubmissionBundleDurablePutResult( + put_attempt_id=uuid4(), + pre_submit_evidence_set_id=uuid4(), + operation_identity=_sha("9"), + admission_id=None, + status="prepared", + replayed=True, + ) + + assert PreparedSubmissionBundlePreparationCommand._result(durable) == ( + SubmissionBundlePreparationResult( + put_attempt_id=durable.put_attempt_id, + admission_id=None, + status="prepared", + replayed=True, + ) + ) + + def _capability(prepared, evidence_set_id): service = PreSubmitEvidenceService(SimpleNamespace()) return service._mint_pass_capability( From 6ad6937aa41186a6dc9d140dba322a099de53469 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sun, 9 Aug 2026 05:11:18 +0100 Subject: [PATCH 5/7] Reconcile ART admission with protected ownership --- ...S-ART-001-04C2-external-review-response.md | 6 + backend/app/adapters/artifacts/__init__.py | 16 +- backend/app/modules/artifacts/service.py | 2 +- .../modules/artifacts/submission_admission.py | 541 +++++++++++++++++- .../submission_admission_publication.py | 214 ------- .../artifacts/submission_preparation.py | 350 ----------- backend/app/modules/tasks/router.py | 2 +- .../test_default_pre_submit_execution.py | 2 +- .../tests/test_submission_bundle_admission.py | 4 +- 9 files changed, 552 insertions(+), 585 deletions(-) delete mode 100644 backend/app/modules/artifacts/submission_admission_publication.py delete mode 100644 backend/app/modules/artifacts/submission_preparation.py diff --git a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-04C2-external-review-response.md b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-04C2-external-review-response.md index df6a6efe4..0fcbd972e 100644 --- a/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-04C2-external-review-response.md +++ b/.agent-loop/initiatives/WS-ART-001-immutable-artifact-storage/reviews/WS-ART-001-04C2-external-review-response.md @@ -45,3 +45,9 @@ None. AUTH activation and admission consumption remain separate reviewed chunks. The route remains hidden and unavailable in this PR. + +WS-QUAL-002-01 merged its protected exact module partition while this ART chunk +was in flight. The ready-admission publisher and hidden preparation command are +therefore consolidated in `submission_admission.py`; the QUAL-owned partition +is unchanged. Do not re-split those classes into new eligible Python modules +until a QUAL-owned protected-partition update has merged. diff --git a/backend/app/adapters/artifacts/__init__.py b/backend/app/adapters/artifacts/__init__.py index 73231930c..02b6ef0ec 100644 --- a/backend/app/adapters/artifacts/__init__.py +++ b/backend/app/adapters/artifacts/__init__.py @@ -30,7 +30,7 @@ from app.modules.artifacts.submission_authorization import ( SubmissionBundlePreparationAuthorization, ) -from app.modules.artifacts.submission_preparation import SubmissionBundlePreparationCommand +from app.modules.artifacts.submission_admission import SubmissionBundlePreparationCommand from app.modules.artifacts.schemas import ( ArtifactInternalAuthority, ) @@ -121,12 +121,8 @@ def submission_archive_limits(settings: Settings) -> SubmissionArchiveLimits: ), maximum_entry_bytes=settings.artifact_submission_zip_maximum_entry_bytes, maximum_expanded_bytes=settings.artifact_submission_zip_maximum_expanded_bytes, - maximum_compression_ratio=( - settings.artifact_submission_zip_maximum_compression_ratio - ), - maximum_inspection_seconds=( - settings.artifact_submission_zip_maximum_inspection_seconds - ), + maximum_compression_ratio=(settings.artifact_submission_zip_maximum_compression_ratio), + maximum_inspection_seconds=(settings.artifact_submission_zip_maximum_inspection_seconds), ) @@ -249,7 +245,7 @@ def get_submission_bundle_preparation_command( PreparedBundleMaterializationService, PreparedBundlePreSubmitEvidenceService, ) - from app.modules.artifacts.submission_preparation import ( + from app.modules.artifacts.submission_admission import ( PreparedSubmissionBundlePreparationCommand, SubmissionBundlePreparationRuntime, ) @@ -285,9 +281,7 @@ async def runtime(): materialization = PreparedBundleMaterializationService( authorization=materialization_authority, preparation=preparation, - archive_inspector=SubmissionArchiveInspector( - submission_archive_limits(settings) - ), + archive_inspector=SubmissionArchiveInspector(submission_archive_limits(settings)), catalogue=catalogue, storage_scheme=storage_scheme, ) diff --git a/backend/app/modules/artifacts/service.py b/backend/app/modules/artifacts/service.py index 0a89aee61..b69aad87a 100644 --- a/backend/app/modules/artifacts/service.py +++ b/backend/app/modules/artifacts/service.py @@ -1275,7 +1275,7 @@ async def _complete_verification( ) ) if outcome == "verified": - from app.modules.artifacts.submission_admission_publication import ( + from app.modules.artifacts.submission_admission import ( SubmissionBundleAdmissionPublicationError, SubmissionBundleAdmissionPublisher, ) diff --git a/backend/app/modules/artifacts/submission_admission.py b/backend/app/modules/artifacts/submission_admission.py index e0ba009e1..e6a67c413 100644 --- a/backend/app/modules/artifacts/submission_admission.py +++ b/backend/app/modules/artifacts/submission_admission.py @@ -2,12 +2,34 @@ from __future__ import annotations +from collections.abc import Callable +from contextlib import AbstractAsyncContextManager from dataclasses import dataclass -from uuid import UUID +from typing import Protocol +from uuid import UUID, uuid4 +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession +from app.interfaces.artifact_operations import ( + PreparedBundleMaterializationRequest, + SubmissionBundlePreparationPort, + SubmissionBundlePreparationRequest, +) +from app.modules.artifacts.models import ( + ArtifactContent, + ArtifactOperationReceipt, + ArtifactPutAttempt, + ArtifactPutObservationReceipt, + ArtifactReplica, + ArtifactVerificationJob, + ArtifactVerificationReceipt, + PreSubmitEvidenceSet, + SubmissionBundleAdmission, + SubmissionBundleDurableIntent, +) from app.modules.artifacts.pre_submit_evidence import PreSubmitPassCapability +from app.modules.artifacts.preparation import ArtifactPreparationService from app.modules.artifacts.schemas import ( ArtifactAdmissionResult, SubmissionBundleArtifactAdmissionRequest, @@ -20,8 +42,28 @@ ArtifactStorageOrchestrator, ) from app.modules.artifacts.sources import PreparedArtifact +from app.modules.artifacts.submission_archive import SubmissionArchiveInspector +from app.modules.artifacts.submission_authorization import ( + SubmissionBundlePreparationAuthorization, +) from app.modules.artifacts.submission_custody import SubmissionBundlePreparedCustody +from app.modules.artifacts.submission_manifest import ( + SubmissionCanonicalPredecessor, + build_submission_manifest, + evaluate_submission_change, +) +from app.modules.artifacts.submission_materialization import ( + PreparedBundleMaterializationService, + PreparedBundlePreSubmitEvidenceService, +) from app.modules.authorization.prepared import PreparedAuthorizationHandle +from app.modules.checkers.catalogue import PreSubmissionCheckerCatalogue +from app.modules.checkers.pre_submit_execution import SubmissionPacketView +from app.modules.tasks.pre_submit_context import ( + compile_locked_pre_submit_plan, + load_canonical_submission_version, + load_locked_pre_submit_context, +) @dataclass(frozen=True, slots=True) @@ -114,10 +156,6 @@ async def publish_after_commit( await prepared.close() raise RuntimeError("submission bundle durable transaction is still active") try: - from app.modules.artifacts.submission_admission_publication import ( - current_submission_bundle_admission_id, - ) - admission_id = None if admission.replayed: async with self._session.begin(): @@ -168,3 +206,496 @@ async def publish_after_commit( ) finally: await prepared.close() + + +class SubmissionBundleAdmissionPublicationError(RuntimeError): + """Durable verification lineage cannot publish a trusted ready admission.""" + + +class SubmissionBundleAdmissionPublisher: + """Project verified submission bytes into one idempotent ready admission.""" + + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def publish_verified( + self, *, verification_job_id: str, verification_receipt_id: str + ) -> SubmissionBundleAdmission | None: + """Publish only submission-bundle lineage; ignore other producer types.""" + job = await self._session.scalar( + select(ArtifactVerificationJob) + .where(ArtifactVerificationJob.id == verification_job_id) + .with_for_update(key_share=True) + ) + if job is None: + raise SubmissionBundleAdmissionPublicationError("verification job is unavailable") + attempt = await self._session.scalar( + select(ArtifactPutAttempt) + .where(ArtifactPutAttempt.id == job.originating_put_attempt_id) + .with_for_update(key_share=True) + ) + if attempt is None: + raise SubmissionBundleAdmissionPublicationError("put attempt is unavailable") + if attempt.producer_request_type != "submission_bundle": + return None + intent = await self._session.scalar( + select(SubmissionBundleDurableIntent) + .where(SubmissionBundleDurableIntent.put_attempt_id == attempt.id) + # Serialize publication on the immutable intent. A weaker key-share + # lock lets two verifier transactions both miss the admission and + # race on its uniqueness constraint. + .with_for_update() + ) + if intent is None: + raise SubmissionBundleAdmissionPublicationError("durable intent is unavailable") + existing = await self._session.scalar( + select(SubmissionBundleAdmission) + .where(SubmissionBundleAdmission.durable_intent_id == intent.id) + .with_for_update() + ) + if existing is not None: + return existing + evidence = await self._session.scalar( + select(PreSubmitEvidenceSet) + .where(PreSubmitEvidenceSet.id == intent.pre_submit_evidence_set_id) + .with_for_update(key_share=True) + ) + replica = await self._session.scalar( + select(ArtifactReplica) + .where(ArtifactReplica.id == job.replica_id) + .with_for_update(key_share=True) + ) + receipt = await self._session.scalar( + select(ArtifactVerificationReceipt) + .where(ArtifactVerificationReceipt.id == verification_receipt_id) + .with_for_update(key_share=True) + ) + content = ( + await self._session.scalar( + select(ArtifactContent) + .where(ArtifactContent.id == replica.content_id) + .with_for_update(key_share=True) + ) + if replica is not None + else None + ) + if not self._matches_verified_lineage(evidence, attempt, job, replica, content, receipt): + raise SubmissionBundleAdmissionPublicationError( + "verified submission bundle lineage does not match" + ) + operation_receipt_id, observation_receipt_id = await self._write_receipt_ids(attempt) + assert evidence is not None and replica is not None and content is not None + assert receipt is not None + now = await self._session.scalar(select(func.now())) + admission = SubmissionBundleAdmission( + id=str(uuid4()), + durable_intent_id=intent.id, + pre_submit_evidence_set_id=evidence.id, + put_attempt_id=attempt.id, + artifact_content_id=content.id, + verified_replica_id=replica.id, + verification_receipt_id=receipt.id, + put_operation_receipt_id=operation_receipt_id, + put_observation_receipt_id=observation_receipt_id, + actor_profile_id=evidence.actor_profile_id, + identity_link_id=evidence.identity_link_id, + project_id=evidence.project_id, + task_id=evidence.task_id, + assignment_id=evidence.assignment_id, + predecessor_submission_id=evidence.predecessor_submission_id, + predecessor_submission_version=evidence.predecessor_submission_version, + locked_policy_context_hash=evidence.locked_policy_context_hash, + semantic_manifest_id=evidence.semantic_manifest_id, + semantic_manifest_sha256=evidence.semantic_manifest_sha256, + archive_sha256=evidence.archive_sha256, + archive_byte_count=evidence.archive_byte_count, + status="ready", + ready_at=now, + consumed_at=None, + consumed_by_submission_id=None, + stale_at=None, + stale_reason=None, + ) + self._session.add(admission) + await self._session.flush() + return admission + + @staticmethod + def _matches_verified_lineage(evidence, attempt, job, replica, content, receipt) -> bool: + return bool( + evidence is not None + and evidence.terminal_status == "passed" + and evidence.eligible + and replica is not None + and content is not None + and receipt is not None + and receipt.verification_job_id == job.id + and receipt.execution_generation == job.execution_generation + and receipt.outcome == "verified" + and receipt.observed_sha256 + == attempt.sha256 + == content.sha256 + == evidence.archive_sha256 + and receipt.observed_byte_count + == attempt.byte_count + == content.byte_count + == evidence.archive_byte_count + and job.originating_put_attempt_id == attempt.id + and job.replica_id == replica.id + and attempt.replica_id == replica.id + and replica.content_id == content.id + and replica.verification_state == "verified" + and replica.availability_state == "available" + and replica.integrity_state == "valid" + and attempt.project_id == evidence.project_id + and attempt.task_id == evidence.task_id + and attempt.producer_ref == evidence.actor_profile_id + and attempt.media_type == "application/zip" + ) + + async def _write_receipt_ids( + self, attempt: ArtifactPutAttempt + ) -> tuple[str | None, str | None]: + if attempt.receipt_id is not None: + receipt = await self._session.scalar( + select(ArtifactOperationReceipt).where( + ArtifactOperationReceipt.id == attempt.receipt_id, + ArtifactOperationReceipt.put_attempt_id == attempt.id, + ArtifactOperationReceipt.replica_id == attempt.replica_id, + ArtifactOperationReceipt.outcome == "stored_pending_verification", + ) + ) + if receipt is None: + raise SubmissionBundleAdmissionPublicationError("put receipt lineage is invalid") + return receipt.id, None + observation = await self._session.scalar( + select(ArtifactPutObservationReceipt) + .where( + ArtifactPutObservationReceipt.put_attempt_id == attempt.id, + ArtifactPutObservationReceipt.outcome == "observed_confirmed", + ArtifactPutObservationReceipt.observed_sha256 == attempt.sha256, + ArtifactPutObservationReceipt.observed_byte_count == attempt.byte_count, + ) + .order_by(ArtifactPutObservationReceipt.execution_generation.desc()) + .limit(1) + ) + if observation is None: + raise SubmissionBundleAdmissionPublicationError( + "put observation receipt lineage is invalid" + ) + return None, observation.id + + +async def current_submission_bundle_admission_id( + session: AsyncSession, *, put_attempt_id: UUID +) -> UUID | None: + """Return bounded current state for hidden exact request replay.""" + value = await session.scalar( + select(SubmissionBundleAdmission.id) + .join( + SubmissionBundleDurableIntent, + SubmissionBundleDurableIntent.id == SubmissionBundleAdmission.durable_intent_id, + ) + .where(SubmissionBundleDurableIntent.put_attempt_id == str(put_attempt_id)) + ) + return UUID(value) if value is not None else None + + +class SubmissionBundlePreparationRejected(RuntimeError): + """The complete effective pre-submit execution did not produce passing custody.""" + + +@dataclass(frozen=True, slots=True) +class SubmissionBundlePreparationResult: + put_attempt_id: UUID + admission_id: UUID | None + status: str + replayed: bool + + +@dataclass(frozen=True, slots=True) +class SubmissionBundlePreparationRuntime: + preparation: ArtifactPreparationService + inspector: SubmissionArchiveInspector + catalogue: PreSubmissionCheckerCatalogue + materialization: PreparedBundleMaterializationService + evidence: PreparedBundlePreSubmitEvidenceService + durable_put: SubmissionBundleDurablePutService + + +class SubmissionBundlePreparationCommand(SubmissionBundlePreparationPort, Protocol): + async def prepare( + self, request: SubmissionBundlePreparationRequest + ) -> SubmissionBundlePreparationResult: ... + + +class PreparedSubmissionBundlePreparationCommand: + """Keep every process-local capability within one hidden request.""" + + def __init__( + self, + *, + session: AsyncSession, + authority: SubmissionBundlePreparationAuthorization, + runtime_factory: Callable[ + [], AbstractAsyncContextManager[SubmissionBundlePreparationRuntime] + ], + ) -> None: + self._session = session + self._authority = authority + self._runtime_factory = runtime_factory + + async def prepare( + self, request: SubmissionBundlePreparationRequest + ) -> SubmissionBundlePreparationResult: + if type(request) is not SubmissionBundlePreparationRequest: + raise TypeError("invalid submission bundle preparation request") + prepared = None + try: + await self._authority.preflight( + authorization_context=request.authorization_context, + task_id=request.task_id, + assignment_id=request.assignment_id, + predecessor_submission_id=request.predecessor_submission_id, + idempotency_key=request.idempotency_key, + ) + if request.media_type.partition(";")[0].strip().lower() != "application/zip": + raise SubmissionBundlePreparationRejected("submission_bundle_media_type_invalid") + async with self._runtime_factory() as runtime: + async with self._session.begin(): + locked = await load_locked_pre_submit_context( + self._session, + actor_profile_id=request.authorization_context.actor_profile_id, + identity_link_id=request.authorization_context.identity_link_id, + task_id=request.task_id, + assignment_id=request.assignment_id, + predecessor_submission_id=request.predecessor_submission_id, + include_actor_identity_locks=False, + ) + plan = compile_locked_pre_submit_plan(locked, runtime.catalogue) + predecessor = await self._load_predecessor(request.predecessor_submission_id) + prepared = await runtime.preparation.prepare( + request.byte_source, + media_type="application/zip", + ) + async with self._session.begin(): + materialization_handle = await runtime.materialization.prepare_authorization( + task_id=request.task_id, + assignment_id=request.assignment_id, + submission_artifact_policy_id=locked.effective_policy_id, + checker_policy_id=locked.pre_submit_policy_id, + prepared_artifact=prepared, + effective_plan=plan, + idempotency_key=request.idempotency_key, + ) + inspection = await prepared.inspect(runtime.inspector) + manifest = build_submission_manifest(inspection) + change_gate = evaluate_submission_change( + commitment=prepared.commitment, + manifest=manifest, + predecessor=predecessor, + predecessor_exists=request.predecessor_submission_id is not None, + current_predecessor=predecessor, + ) + materialization_request = PreparedBundleMaterializationRequest( + prepared_authorization=materialization_handle, + task_id=request.task_id, + assignment_id=request.assignment_id, + submission_artifact_policy_id=locked.effective_policy_id, + checker_policy_id=locked.pre_submit_policy_id, + prepared_artifact=prepared, + effective_plan=plan, + inspection=inspection, + manifest=manifest, + change_gate=change_gate, + packet=SubmissionPacketView( + summary=request.summary, + contributor_attestation=request.contributor_attestation, + ), + ) + execution = await runtime.evidence.materialize(materialization_request) + evidence = await runtime.evidence.persist( + materialization_request, + execution=execution, + actor_profile_id=request.authorization_context.actor_profile_id, + identity_link_id=request.authorization_context.identity_link_id, + predecessor_submission_id=request.predecessor_submission_id, + ) + if evidence.pass_capability is None: + replay = await self._existing_durable_result(evidence.evidence.evidence_set_id) + if replay is None: + raise SubmissionBundlePreparationRejected( + "pre_submission_checked_custody_unavailable" + ) + await prepared.close() + prepared = None + return replay + replay_intent_id = await self._matching_replay_intent( + evidence.evidence.evidence_set_id + ) + async with self._authority.transaction(): + final_handle = await self._authority.prepare_final( + authorization_context=request.authorization_context, + task_id=request.task_id, + assignment_id=request.assignment_id, + predecessor_submission_id=request.predecessor_submission_id, + idempotency_key=request.idempotency_key, + ) + retained, _, durable = await runtime.durable_put.admit_in_transaction( + SubmissionBundleDurablePutRequest( + prepared_authorization=final_handle, + prepared_artifact=prepared, + pass_capability=evidence.pass_capability, + replay_durable_intent_id=replay_intent_id, + ) + ) + prepared = None + result = await runtime.durable_put.publish_after_commit( + retained, + evidence.evidence.evidence_set_id, + durable, + ) + return self._result(result) + finally: + if prepared is not None: + await prepared.close() + self._authority.close() + + async def _matching_replay_intent(self, evidence_id: UUID) -> UUID | None: + """Find an older exact lineage without trusting client replay selectors.""" + async with self._session.begin(): + current = await self._session.get(PreSubmitEvidenceSet, str(evidence_id)) + if current is None: + raise SubmissionBundlePreparationRejected( + "pre_submission_checked_custody_unavailable" + ) + fields = ( + "actor_profile_id", + "identity_link_id", + "project_id", + "task_id", + "assignment_id", + "predecessor_submission_id", + "predecessor_submission_version", + "archive_sha256", + "archive_byte_count", + "semantic_manifest_sha256", + "guide_id", + "guide_version", + "source_snapshot_id", + "source_snapshot_sha256", + "locked_guide_sha256", + "effective_policy_id", + "locked_artifact_policy_sha256", + "pre_submit_policy_id", + "locked_checker_policy_sha256", + "effective_plan_sha256", + "catalogue_id", + "catalogue_version", + "catalogue_manifest_sha256", + "storage_scheme", + "terminal_status", + "eligible", + "result_count", + "result_manifest_sha256", + ) + statement = ( + select(SubmissionBundleDurableIntent.id) + .join( + PreSubmitEvidenceSet, + PreSubmitEvidenceSet.id + == SubmissionBundleDurableIntent.pre_submit_evidence_set_id, + ) + .join( + ArtifactPutAttempt, + ArtifactPutAttempt.id == SubmissionBundleDurableIntent.put_attempt_id, + ) + .where( + PreSubmitEvidenceSet.id != current.id, + ArtifactPutAttempt.status.in_( + ( + "prepared", + "acknowledgement_unknown", + "absent_replay_required", + "object_confirmed", + ) + ), + *( + getattr(PreSubmitEvidenceSet, field) == getattr(current, field) + for field in fields + ), + ) + .order_by(SubmissionBundleDurableIntent.created_at) + .limit(1) + ) + value = await self._session.scalar(statement) + return UUID(value) if value is not None else None + + async def _existing_durable_result( + self, evidence_id: UUID + ) -> SubmissionBundlePreparationResult | None: + async with self._session.begin(): + row = ( + await self._session.execute( + select( + SubmissionBundleDurableIntent, + ArtifactPutAttempt, + SubmissionBundleAdmission, + ) + .join( + ArtifactPutAttempt, + ArtifactPutAttempt.id == SubmissionBundleDurableIntent.put_attempt_id, + ) + .outerjoin( + SubmissionBundleAdmission, + SubmissionBundleAdmission.durable_intent_id + == SubmissionBundleDurableIntent.id, + ) + .where( + SubmissionBundleDurableIntent.pre_submit_evidence_set_id == str(evidence_id) + ) + ) + ).one_or_none() + if row is None: + return None + _, attempt, admission = row + return SubmissionBundlePreparationResult( + put_attempt_id=UUID(attempt.id), + admission_id=UUID(admission.id) if admission is not None else None, + status="ready" if admission is not None else attempt.status, + replayed=True, + ) + + async def _load_predecessor( + self, submission_id: UUID | None + ) -> SubmissionCanonicalPredecessor | None: + if submission_id is None: + return None + admission = await self._session.scalar( + select(SubmissionBundleAdmission).where( + SubmissionBundleAdmission.consumed_by_submission_id == str(submission_id), + SubmissionBundleAdmission.status == "consumed", + ) + ) + version = await load_canonical_submission_version( + self._session, submission_id=submission_id + ) + if admission is None or version is None: + raise SubmissionBundlePreparationRejected( + "submission_canonical_predecessor_unavailable" + ) + return SubmissionCanonicalPredecessor( + submission_id=submission_id, + submission_version=version, + archive_sha256=admission.archive_sha256, + semantic_manifest_sha256=admission.semantic_manifest_sha256, + ) + + @staticmethod + def _result(result: SubmissionBundleDurablePutResult) -> SubmissionBundlePreparationResult: + return SubmissionBundlePreparationResult( + put_attempt_id=result.put_attempt_id, + admission_id=result.admission_id, + status=result.status, + replayed=result.replayed, + ) diff --git a/backend/app/modules/artifacts/submission_admission_publication.py b/backend/app/modules/artifacts/submission_admission_publication.py deleted file mode 100644 index fa04c1fdb..000000000 --- a/backend/app/modules/artifacts/submission_admission_publication.py +++ /dev/null @@ -1,214 +0,0 @@ -"""Verified-only publication of immutable ready submission-bundle admissions.""" - -from __future__ import annotations - -from uuid import UUID, uuid4 - -from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.modules.artifacts.models import ( - ArtifactContent, - ArtifactOperationReceipt, - ArtifactPutAttempt, - ArtifactPutObservationReceipt, - ArtifactReplica, - ArtifactVerificationJob, - ArtifactVerificationReceipt, - PreSubmitEvidenceSet, - SubmissionBundleAdmission, - SubmissionBundleDurableIntent, -) - - -class SubmissionBundleAdmissionPublicationError(RuntimeError): - """Durable verification lineage cannot publish a trusted ready admission.""" - - -class SubmissionBundleAdmissionPublisher: - """Project verified submission bytes into one idempotent ready admission.""" - - def __init__(self, session: AsyncSession) -> None: - self._session = session - - async def publish_verified( - self, *, verification_job_id: str, verification_receipt_id: str - ) -> SubmissionBundleAdmission | None: - """Publish only submission-bundle lineage; ignore other producer types.""" - job = await self._session.scalar( - select(ArtifactVerificationJob) - .where(ArtifactVerificationJob.id == verification_job_id) - .with_for_update(key_share=True) - ) - if job is None: - raise SubmissionBundleAdmissionPublicationError("verification job is unavailable") - attempt = await self._session.scalar( - select(ArtifactPutAttempt) - .where(ArtifactPutAttempt.id == job.originating_put_attempt_id) - .with_for_update(key_share=True) - ) - if attempt is None: - raise SubmissionBundleAdmissionPublicationError("put attempt is unavailable") - if attempt.producer_request_type != "submission_bundle": - return None - intent = await self._session.scalar( - select(SubmissionBundleDurableIntent) - .where(SubmissionBundleDurableIntent.put_attempt_id == attempt.id) - # Serialize publication on the immutable intent. A weaker key-share - # lock lets two verifier transactions both miss the admission and - # race on its uniqueness constraint. - .with_for_update() - ) - if intent is None: - raise SubmissionBundleAdmissionPublicationError("durable intent is unavailable") - existing = await self._session.scalar( - select(SubmissionBundleAdmission) - .where(SubmissionBundleAdmission.durable_intent_id == intent.id) - .with_for_update() - ) - if existing is not None: - return existing - evidence = await self._session.scalar( - select(PreSubmitEvidenceSet) - .where(PreSubmitEvidenceSet.id == intent.pre_submit_evidence_set_id) - .with_for_update(key_share=True) - ) - replica = await self._session.scalar( - select(ArtifactReplica) - .where(ArtifactReplica.id == job.replica_id) - .with_for_update(key_share=True) - ) - receipt = await self._session.scalar( - select(ArtifactVerificationReceipt) - .where(ArtifactVerificationReceipt.id == verification_receipt_id) - .with_for_update(key_share=True) - ) - content = ( - await self._session.scalar( - select(ArtifactContent) - .where(ArtifactContent.id == replica.content_id) - .with_for_update(key_share=True) - ) - if replica is not None - else None - ) - if not self._matches_verified_lineage(evidence, attempt, job, replica, content, receipt): - raise SubmissionBundleAdmissionPublicationError( - "verified submission bundle lineage does not match" - ) - operation_receipt_id, observation_receipt_id = await self._write_receipt_ids(attempt) - assert evidence is not None and replica is not None and content is not None - assert receipt is not None - now = await self._session.scalar(select(func.now())) - admission = SubmissionBundleAdmission( - id=str(uuid4()), - durable_intent_id=intent.id, - pre_submit_evidence_set_id=evidence.id, - put_attempt_id=attempt.id, - artifact_content_id=content.id, - verified_replica_id=replica.id, - verification_receipt_id=receipt.id, - put_operation_receipt_id=operation_receipt_id, - put_observation_receipt_id=observation_receipt_id, - actor_profile_id=evidence.actor_profile_id, - identity_link_id=evidence.identity_link_id, - project_id=evidence.project_id, - task_id=evidence.task_id, - assignment_id=evidence.assignment_id, - predecessor_submission_id=evidence.predecessor_submission_id, - predecessor_submission_version=evidence.predecessor_submission_version, - locked_policy_context_hash=evidence.locked_policy_context_hash, - semantic_manifest_id=evidence.semantic_manifest_id, - semantic_manifest_sha256=evidence.semantic_manifest_sha256, - archive_sha256=evidence.archive_sha256, - archive_byte_count=evidence.archive_byte_count, - status="ready", - ready_at=now, - consumed_at=None, - consumed_by_submission_id=None, - stale_at=None, - stale_reason=None, - ) - self._session.add(admission) - await self._session.flush() - return admission - - @staticmethod - def _matches_verified_lineage(evidence, attempt, job, replica, content, receipt) -> bool: - return bool( - evidence is not None - and evidence.terminal_status == "passed" - and evidence.eligible - and replica is not None - and content is not None - and receipt is not None - and receipt.verification_job_id == job.id - and receipt.execution_generation == job.execution_generation - and receipt.outcome == "verified" - and receipt.observed_sha256 - == attempt.sha256 - == content.sha256 - == evidence.archive_sha256 - and receipt.observed_byte_count - == attempt.byte_count - == content.byte_count - == evidence.archive_byte_count - and job.originating_put_attempt_id == attempt.id - and job.replica_id == replica.id - and attempt.replica_id == replica.id - and replica.content_id == content.id - and replica.verification_state == "verified" - and replica.availability_state == "available" - and replica.integrity_state == "valid" - and attempt.project_id == evidence.project_id - and attempt.task_id == evidence.task_id - and attempt.producer_ref == evidence.actor_profile_id - and attempt.media_type == "application/zip" - ) - - async def _write_receipt_ids( - self, attempt: ArtifactPutAttempt - ) -> tuple[str | None, str | None]: - if attempt.receipt_id is not None: - receipt = await self._session.scalar( - select(ArtifactOperationReceipt).where( - ArtifactOperationReceipt.id == attempt.receipt_id, - ArtifactOperationReceipt.put_attempt_id == attempt.id, - ArtifactOperationReceipt.replica_id == attempt.replica_id, - ArtifactOperationReceipt.outcome == "stored_pending_verification", - ) - ) - if receipt is None: - raise SubmissionBundleAdmissionPublicationError("put receipt lineage is invalid") - return receipt.id, None - observation = await self._session.scalar( - select(ArtifactPutObservationReceipt) - .where( - ArtifactPutObservationReceipt.put_attempt_id == attempt.id, - ArtifactPutObservationReceipt.outcome == "observed_confirmed", - ArtifactPutObservationReceipt.observed_sha256 == attempt.sha256, - ArtifactPutObservationReceipt.observed_byte_count == attempt.byte_count, - ) - .order_by(ArtifactPutObservationReceipt.execution_generation.desc()) - .limit(1) - ) - if observation is None: - raise SubmissionBundleAdmissionPublicationError( - "put observation receipt lineage is invalid" - ) - return None, observation.id - - -async def current_submission_bundle_admission_id( - session: AsyncSession, *, put_attempt_id: UUID -) -> UUID | None: - """Return bounded current state for hidden exact request replay.""" - value = await session.scalar( - select(SubmissionBundleAdmission.id) - .join( - SubmissionBundleDurableIntent, - SubmissionBundleDurableIntent.id == SubmissionBundleAdmission.durable_intent_id, - ) - .where(SubmissionBundleDurableIntent.put_attempt_id == str(put_attempt_id)) - ) - return UUID(value) if value is not None else None diff --git a/backend/app/modules/artifacts/submission_preparation.py b/backend/app/modules/artifacts/submission_preparation.py deleted file mode 100644 index 8504c0d9f..000000000 --- a/backend/app/modules/artifacts/submission_preparation.py +++ /dev/null @@ -1,350 +0,0 @@ -"""Hidden continuous contributor ZIP preparation orchestration.""" - -from __future__ import annotations - -from collections.abc import Callable -from contextlib import AbstractAsyncContextManager -from dataclasses import dataclass -from typing import Protocol -from uuid import UUID - -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.interfaces.artifact_operations import ( - PreparedBundleMaterializationRequest, - SubmissionBundlePreparationPort, - SubmissionBundlePreparationRequest, -) -from app.modules.artifacts.models import ( - ArtifactPutAttempt, - PreSubmitEvidenceSet, - SubmissionBundleAdmission, - SubmissionBundleDurableIntent, -) -from app.modules.artifacts.preparation import ArtifactPreparationService -from app.modules.artifacts.submission_admission import ( - SubmissionBundleDurablePutRequest, - SubmissionBundleDurablePutResult, - SubmissionBundleDurablePutService, -) -from app.modules.artifacts.submission_archive import SubmissionArchiveInspector -from app.modules.artifacts.submission_authorization import ( - SubmissionBundlePreparationAuthorization, -) -from app.modules.artifacts.submission_manifest import ( - SubmissionCanonicalPredecessor, - build_submission_manifest, - evaluate_submission_change, -) -from app.modules.artifacts.submission_materialization import ( - PreparedBundleMaterializationService, - PreparedBundlePreSubmitEvidenceService, -) -from app.modules.checkers.catalogue import PreSubmissionCheckerCatalogue -from app.modules.checkers.pre_submit_execution import SubmissionPacketView -from app.modules.tasks.pre_submit_context import ( - compile_locked_pre_submit_plan, - load_canonical_submission_version, - load_locked_pre_submit_context, -) - - -class SubmissionBundlePreparationRejected(RuntimeError): - """The complete effective pre-submit execution did not produce passing custody.""" - - -@dataclass(frozen=True, slots=True) -class SubmissionBundlePreparationResult: - put_attempt_id: UUID - admission_id: UUID | None - status: str - replayed: bool - - -@dataclass(frozen=True, slots=True) -class SubmissionBundlePreparationRuntime: - preparation: ArtifactPreparationService - inspector: SubmissionArchiveInspector - catalogue: PreSubmissionCheckerCatalogue - materialization: PreparedBundleMaterializationService - evidence: PreparedBundlePreSubmitEvidenceService - durable_put: SubmissionBundleDurablePutService - - -class SubmissionBundlePreparationCommand(SubmissionBundlePreparationPort, Protocol): - async def prepare( - self, request: SubmissionBundlePreparationRequest - ) -> SubmissionBundlePreparationResult: ... - - -class PreparedSubmissionBundlePreparationCommand: - """Keep every process-local capability within one hidden request.""" - - def __init__( - self, - *, - session: AsyncSession, - authority: SubmissionBundlePreparationAuthorization, - runtime_factory: Callable[ - [], AbstractAsyncContextManager[SubmissionBundlePreparationRuntime] - ], - ) -> None: - self._session = session - self._authority = authority - self._runtime_factory = runtime_factory - - async def prepare( - self, request: SubmissionBundlePreparationRequest - ) -> SubmissionBundlePreparationResult: - if type(request) is not SubmissionBundlePreparationRequest: - raise TypeError("invalid submission bundle preparation request") - prepared = None - try: - await self._authority.preflight( - authorization_context=request.authorization_context, - task_id=request.task_id, - assignment_id=request.assignment_id, - predecessor_submission_id=request.predecessor_submission_id, - idempotency_key=request.idempotency_key, - ) - if request.media_type.partition(";")[0].strip().lower() != "application/zip": - raise SubmissionBundlePreparationRejected("submission_bundle_media_type_invalid") - async with self._runtime_factory() as runtime: - async with self._session.begin(): - locked = await load_locked_pre_submit_context( - self._session, - actor_profile_id=request.authorization_context.actor_profile_id, - identity_link_id=request.authorization_context.identity_link_id, - task_id=request.task_id, - assignment_id=request.assignment_id, - predecessor_submission_id=request.predecessor_submission_id, - include_actor_identity_locks=False, - ) - plan = compile_locked_pre_submit_plan(locked, runtime.catalogue) - predecessor = await self._load_predecessor(request.predecessor_submission_id) - prepared = await runtime.preparation.prepare( - request.byte_source, - media_type="application/zip", - ) - async with self._session.begin(): - materialization_handle = await runtime.materialization.prepare_authorization( - task_id=request.task_id, - assignment_id=request.assignment_id, - submission_artifact_policy_id=locked.effective_policy_id, - checker_policy_id=locked.pre_submit_policy_id, - prepared_artifact=prepared, - effective_plan=plan, - idempotency_key=request.idempotency_key, - ) - inspection = await prepared.inspect(runtime.inspector) - manifest = build_submission_manifest(inspection) - change_gate = evaluate_submission_change( - commitment=prepared.commitment, - manifest=manifest, - predecessor=predecessor, - predecessor_exists=request.predecessor_submission_id is not None, - current_predecessor=predecessor, - ) - materialization_request = PreparedBundleMaterializationRequest( - prepared_authorization=materialization_handle, - task_id=request.task_id, - assignment_id=request.assignment_id, - submission_artifact_policy_id=locked.effective_policy_id, - checker_policy_id=locked.pre_submit_policy_id, - prepared_artifact=prepared, - effective_plan=plan, - inspection=inspection, - manifest=manifest, - change_gate=change_gate, - packet=SubmissionPacketView( - summary=request.summary, - contributor_attestation=request.contributor_attestation, - ), - ) - execution = await runtime.evidence.materialize(materialization_request) - evidence = await runtime.evidence.persist( - materialization_request, - execution=execution, - actor_profile_id=request.authorization_context.actor_profile_id, - identity_link_id=request.authorization_context.identity_link_id, - predecessor_submission_id=request.predecessor_submission_id, - ) - if evidence.pass_capability is None: - replay = await self._existing_durable_result(evidence.evidence.evidence_set_id) - if replay is None: - raise SubmissionBundlePreparationRejected( - "pre_submission_checked_custody_unavailable" - ) - await prepared.close() - prepared = None - return replay - replay_intent_id = await self._matching_replay_intent( - evidence.evidence.evidence_set_id - ) - async with self._authority.transaction(): - final_handle = await self._authority.prepare_final( - authorization_context=request.authorization_context, - task_id=request.task_id, - assignment_id=request.assignment_id, - predecessor_submission_id=request.predecessor_submission_id, - idempotency_key=request.idempotency_key, - ) - retained, _, durable = await runtime.durable_put.admit_in_transaction( - SubmissionBundleDurablePutRequest( - prepared_authorization=final_handle, - prepared_artifact=prepared, - pass_capability=evidence.pass_capability, - replay_durable_intent_id=replay_intent_id, - ) - ) - prepared = None - result = await runtime.durable_put.publish_after_commit( - retained, - evidence.evidence.evidence_set_id, - durable, - ) - return self._result(result) - finally: - if prepared is not None: - await prepared.close() - self._authority.close() - - async def _matching_replay_intent(self, evidence_id: UUID) -> UUID | None: - """Find an older exact lineage without trusting client replay selectors.""" - async with self._session.begin(): - current = await self._session.get(PreSubmitEvidenceSet, str(evidence_id)) - if current is None: - raise SubmissionBundlePreparationRejected( - "pre_submission_checked_custody_unavailable" - ) - fields = ( - "actor_profile_id", - "identity_link_id", - "project_id", - "task_id", - "assignment_id", - "predecessor_submission_id", - "predecessor_submission_version", - "archive_sha256", - "archive_byte_count", - "semantic_manifest_sha256", - "guide_id", - "guide_version", - "source_snapshot_id", - "source_snapshot_sha256", - "locked_guide_sha256", - "effective_policy_id", - "locked_artifact_policy_sha256", - "pre_submit_policy_id", - "locked_checker_policy_sha256", - "effective_plan_sha256", - "catalogue_id", - "catalogue_version", - "catalogue_manifest_sha256", - "storage_scheme", - "terminal_status", - "eligible", - "result_count", - "result_manifest_sha256", - ) - statement = ( - select(SubmissionBundleDurableIntent.id) - .join( - PreSubmitEvidenceSet, - PreSubmitEvidenceSet.id - == SubmissionBundleDurableIntent.pre_submit_evidence_set_id, - ) - .join( - ArtifactPutAttempt, - ArtifactPutAttempt.id == SubmissionBundleDurableIntent.put_attempt_id, - ) - .where( - PreSubmitEvidenceSet.id != current.id, - ArtifactPutAttempt.status.in_( - ( - "prepared", - "acknowledgement_unknown", - "absent_replay_required", - "object_confirmed", - ) - ), - *( - getattr(PreSubmitEvidenceSet, field) == getattr(current, field) - for field in fields - ), - ) - .order_by(SubmissionBundleDurableIntent.created_at) - .limit(1) - ) - value = await self._session.scalar(statement) - return UUID(value) if value is not None else None - - async def _existing_durable_result( - self, evidence_id: UUID - ) -> SubmissionBundlePreparationResult | None: - async with self._session.begin(): - row = ( - await self._session.execute( - select( - SubmissionBundleDurableIntent, - ArtifactPutAttempt, - SubmissionBundleAdmission, - ) - .join( - ArtifactPutAttempt, - ArtifactPutAttempt.id == SubmissionBundleDurableIntent.put_attempt_id, - ) - .outerjoin( - SubmissionBundleAdmission, - SubmissionBundleAdmission.durable_intent_id - == SubmissionBundleDurableIntent.id, - ) - .where( - SubmissionBundleDurableIntent.pre_submit_evidence_set_id == str(evidence_id) - ) - ) - ).one_or_none() - if row is None: - return None - _, attempt, admission = row - return SubmissionBundlePreparationResult( - put_attempt_id=UUID(attempt.id), - admission_id=UUID(admission.id) if admission is not None else None, - status="ready" if admission is not None else attempt.status, - replayed=True, - ) - - async def _load_predecessor( - self, submission_id: UUID | None - ) -> SubmissionCanonicalPredecessor | None: - if submission_id is None: - return None - admission = await self._session.scalar( - select(SubmissionBundleAdmission).where( - SubmissionBundleAdmission.consumed_by_submission_id == str(submission_id), - SubmissionBundleAdmission.status == "consumed", - ) - ) - version = await load_canonical_submission_version( - self._session, submission_id=submission_id - ) - if admission is None or version is None: - raise SubmissionBundlePreparationRejected( - "submission_canonical_predecessor_unavailable" - ) - return SubmissionCanonicalPredecessor( - submission_id=submission_id, - submission_version=version, - archive_sha256=admission.archive_sha256, - semantic_manifest_sha256=admission.semantic_manifest_sha256, - ) - - @staticmethod - def _result(result: SubmissionBundleDurablePutResult) -> SubmissionBundlePreparationResult: - return SubmissionBundlePreparationResult( - put_attempt_id=result.put_attempt_id, - admission_id=result.admission_id, - status=result.status, - replayed=result.replayed, - ) diff --git a/backend/app/modules/tasks/router.py b/backend/app/modules/tasks/router.py index 5ad19f1ae..1f1e62d72 100644 --- a/backend/app/modules/tasks/router.py +++ b/backend/app/modules/tasks/router.py @@ -18,7 +18,7 @@ from app.interfaces.artifact_operations import SubmissionBundlePreparationRequest from app.modules.artifacts.authorization import get_artifact_authorization_context from app.modules.artifacts.schemas import ArtifactAuthorityDeniedError -from app.modules.artifacts.submission_preparation import ( +from app.modules.artifacts.submission_admission import ( SubmissionBundlePreparationCommand, SubmissionBundlePreparationRejected, ) diff --git a/backend/tests/test_default_pre_submit_execution.py b/backend/tests/test_default_pre_submit_execution.py index 3368a8c9d..d33e086d9 100644 --- a/backend/tests/test_default_pre_submit_execution.py +++ b/backend/tests/test_default_pre_submit_execution.py @@ -47,7 +47,7 @@ SubmissionBundleDurablePutRequest, SubmissionBundleDurablePutService, ) -from app.modules.artifacts.submission_admission_publication import ( +from app.modules.artifacts.submission_admission import ( SubmissionBundleAdmissionPublisher, ) from app.modules.artifacts.operator import ArtifactOperatorService diff --git a/backend/tests/test_submission_bundle_admission.py b/backend/tests/test_submission_bundle_admission.py index 49af275f8..0c5efb907 100644 --- a/backend/tests/test_submission_bundle_admission.py +++ b/backend/tests/test_submission_bundle_admission.py @@ -32,13 +32,13 @@ SubmissionBundleDurablePutResult, SubmissionBundleDurablePutService, ) -from app.modules.artifacts.submission_admission_publication import ( +from app.modules.artifacts.submission_admission import ( SubmissionBundleAdmissionPublicationError, SubmissionBundleAdmissionPublisher, ) from app.modules.artifacts.service import ArtifactAdmissionService from app.modules.artifacts.submission_custody import SubmissionBundlePreparedCustody -from app.modules.artifacts.submission_preparation import ( +from app.modules.artifacts.submission_admission import ( PreparedSubmissionBundlePreparationCommand, SubmissionBundlePreparationRejected, SubmissionBundlePreparationResult, From a69a937b2336cc56383e15f20640199dd5c6335d Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sun, 9 Aug 2026 05:27:45 +0100 Subject: [PATCH 6/7] Cover checked custody replay orchestration --- .../tests/test_submission_bundle_admission.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/backend/tests/test_submission_bundle_admission.py b/backend/tests/test_submission_bundle_admission.py index 0c5efb907..313ea762a 100644 --- a/backend/tests/test_submission_bundle_admission.py +++ b/backend/tests/test_submission_bundle_admission.py @@ -12,6 +12,7 @@ import pytest from fastapi import HTTPException +import app.modules.artifacts.submission_admission as submission_admission_module from app.modules.artifacts.pre_submit_evidence import ( PreSubmitEvidenceConflict, PreSubmitEvidenceService, @@ -212,6 +213,104 @@ async def test_existing_durable_preparation_projects_exact_ready_admission() -> ) +@pytest.mark.asyncio +async def test_hidden_preparation_replays_persisted_checked_custody(monkeypatch) -> None: + actor_id = uuid4() + task_id = uuid4() + assignment_id = uuid4() + evidence_id = uuid4() + expected = SubmissionBundlePreparationResult( + put_attempt_id=uuid4(), + admission_id=uuid4(), + status="ready", + replayed=True, + ) + locked = SimpleNamespace(effective_policy_id=uuid4(), pre_submit_policy_id=uuid4()) + prepared = SimpleNamespace( + commitment=object(), + inspect=AsyncMock(return_value=object()), + close=AsyncMock(), + ) + runtime = SimpleNamespace( + preparation=SimpleNamespace(prepare=AsyncMock(return_value=prepared)), + inspector=object(), + catalogue=object(), + materialization=SimpleNamespace(prepare_authorization=AsyncMock(return_value=object())), + evidence=SimpleNamespace( + materialize=AsyncMock(return_value=object()), + persist=AsyncMock( + return_value=SimpleNamespace( + evidence=SimpleNamespace(evidence_set_id=evidence_id), + pass_capability=None, + ) + ), + ), + durable_put=object(), + ) + + @asynccontextmanager + async def runtime_factory(): + yield runtime + + monkeypatch.setattr( + submission_admission_module, + "load_locked_pre_submit_context", + AsyncMock(return_value=locked), + ) + monkeypatch.setattr( + submission_admission_module, + "compile_locked_pre_submit_plan", + Mock(return_value=object()), + ) + monkeypatch.setattr( + submission_admission_module, + "build_submission_manifest", + Mock(return_value=object()), + ) + monkeypatch.setattr( + submission_admission_module, + "evaluate_submission_change", + Mock(return_value=object()), + ) + authority = SimpleNamespace(preflight=AsyncMock(), close=Mock()) + command = PreparedSubmissionBundlePreparationCommand( + session=SimpleNamespace(begin=_transaction), + authority=authority, + runtime_factory=runtime_factory, + ) + command._load_predecessor = AsyncMock(return_value=None) + command._existing_durable_result = AsyncMock(return_value=expected) + + result = await command.prepare( + SubmissionBundlePreparationRequest( + authorization_context=HumanAuthorizationContext( + actor_profile_id=actor_id, + actor_kind=ActorKind.HUMAN, + actor_status=ActorStatus.ACTIVE, + identity_link_id=uuid4(), + identity_link_status=IdentityLinkStatus.ACTIVE, + request_id=uuid4(), + correlation_id=uuid4(), + ), + task_id=task_id, + assignment_id=assignment_id, + predecessor_submission_id=None, + idempotency_key=uuid4(), + summary="summary", + contributor_attestation="attestation", + media_type="application/zip", + byte_source=artifact_byte_stream(b"PK\x03\x04replay"), + ) + ) + + assert result == expected + runtime.preparation.prepare.assert_awaited_once() + runtime.evidence.materialize.assert_awaited_once() + runtime.evidence.persist.assert_awaited_once() + prepared.close.assert_awaited_once() + authority.close.assert_called_once_with() + + def test_durable_put_result_projects_without_losing_replay_state() -> None: durable = SubmissionBundleDurablePutResult( put_attempt_id=uuid4(), From ea614953f2b29437de37298c0701a0b721dfd359 Mon Sep 17 00:00:00 2001 From: Abiorh001 Date: Sun, 9 Aug 2026 05:43:28 +0100 Subject: [PATCH 7/7] Cover preparation context conflict mapping --- .../tests/test_submission_bundle_admission.py | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/backend/tests/test_submission_bundle_admission.py b/backend/tests/test_submission_bundle_admission.py index 313ea762a..164c70981 100644 --- a/backend/tests/test_submission_bundle_admission.py +++ b/backend/tests/test_submission_bundle_admission.py @@ -11,6 +11,7 @@ import pytest from fastapi import HTTPException +from starlette.requests import Request import app.modules.artifacts.submission_admission as submission_admission_module from app.modules.artifacts.pre_submit_evidence import ( @@ -56,8 +57,10 @@ from app.main import create_app from app.modules.tasks.router import ( _require_ascii_submission_packet_headers, + prepare_submission_bundle, router as tasks_router, ) +from app.modules.tasks.pre_submit_context import PreSubmitLockedContextInvalid def _sha(character: str) -> str: @@ -87,6 +90,45 @@ def test_submission_packet_headers_reject_non_ascii() -> None: assert failure.value.detail == "submission_bundle_packet_header_encoding_invalid" +@pytest.mark.asyncio +async def test_hidden_preparation_maps_locked_context_race_to_bounded_conflict() -> None: + command = SimpleNamespace( + prepare=AsyncMock(side_effect=PreSubmitLockedContextInvalid("changed")) + ) + context = HumanAuthorizationContext( + actor_profile_id=uuid4(), + actor_kind=ActorKind.HUMAN, + actor_status=ActorStatus.ACTIVE, + identity_link_id=uuid4(), + identity_link_status=IdentityLinkStatus.ACTIVE, + request_id=uuid4(), + correlation_id=uuid4(), + ) + request = Request( + { + "type": "http", + "method": "POST", + "path": "/", + "headers": [(b"content-type", b"application/zip")], + } + ) + + with pytest.raises(HTTPException) as failure: + await prepare_submission_bundle( + task_id=str(uuid4()), + request=request, + context=context, + command=command, + assignment_id=str(uuid4()), + idempotency_key=str(uuid4()), + summary="summary", + contributor_attestation="attestation", + ) + + assert failure.value.status_code == 409 + assert failure.value.detail == "submission_bundle_preparation_context_changed" + + @asynccontextmanager async def _transaction(): yield