From 79e1be082ddf336ddfbebd5c3c3d663f0cd7574c Mon Sep 17 00:00:00 2001 From: Aman Singh Date: Tue, 15 Sep 2026 15:35:12 +0530 Subject: [PATCH] fix(camera_video): stage scratch-bound artifacts to durable artifacts/ directory When camera_video (and other enrichments) produce artifacts in the scratch directory, the publish loop was treating them as durable. On reprocess, scratch is cleaned and these artifacts become stale or missing. Now the publish loop detects if an artifact is under scratch_dir and copies it to run_dir/artifacts/ before publishing, ensuring the URI remains valid across reprocess cycles. Also updates the test assertion in test_camera_video.py to verify that published artifacts are NOT under scratch. --- src/hflow/app.py | 37 ++++++++++++++++++++++++++++++++++--- tests/test_camera_video.py | 5 ++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/src/hflow/app.py b/src/hflow/app.py index fffa7ec9..c6a3c0c9 100644 --- a/src/hflow/app.py +++ b/src/hflow/app.py @@ -2716,22 +2716,53 @@ def render_contact_sheets(media_episode: Episode) -> EnrichmentResult: continue for artifact_name, artifact_path in enrichment_result.artifacts.items(): resolved_artifact_path = artifact_path.resolve() + # Artifact under scratch_dir is transient; stage it to artifacts/ + # so the published URI remains valid across reprocess. + in_scratch = False try: - artifact_relative_path = resolved_artifact_path.relative_to(run_dir.resolve()) - artifact_key = artifact_relative_path.as_posix() + resolved_artifact_path.relative_to(scratch_dir.resolve()) + in_scratch = True except ValueError: + pass + if in_scratch: step_directory = ( f"{_sanitize_topic(enrichment_run.enrichment.name)}-" f"{enrichment_run.enrichment.version}" ) artifact_name_digest = hashlib.sha256(artifact_name.encode()).hexdigest()[:8] + # Stage the artifact to a stable location within run_dir. + staging_dir = run_dir / "artifacts" / step_directory + staging_dir.mkdir(parents=True, exist_ok=True) + staged_artifact_path = staging_dir / artifact_path.name + if not staged_artifact_path.exists(): + shutil.copy2(resolved_artifact_path, staged_artifact_path) + publish_path = staged_artifact_path artifact_key = ( f"artifacts/{step_directory}/{_sanitize_topic(artifact_name)}-" f"{artifact_name_digest}/{artifact_path.name}" ) + else: + try: + artifact_relative_path = resolved_artifact_path.relative_to( + run_dir.resolve() + ) + artifact_key = artifact_relative_path.as_posix() + except ValueError: + step_directory = ( + f"{_sanitize_topic(enrichment_run.enrichment.name)}-" + f"{enrichment_run.enrichment.version}" + ) + artifact_name_digest = hashlib.sha256(artifact_name.encode()).hexdigest()[ + :8 + ] + artifact_key = ( + f"artifacts/{step_directory}/{_sanitize_topic(artifact_name)}-" + f"{artifact_name_digest}/{artifact_path.name}" + ) + publish_path = artifact_path try: enrichment_run.artifact_uris[artifact_name] = run_storage_root.publish( - artifact_path, artifact_key + publish_path, artifact_key ) except Exception as error: # A missing or unreadable artifact file is the STEP's diff --git a/tests/test_camera_video.py b/tests/test_camera_video.py index 715921d8..60a486f3 100644 --- a/tests/test_camera_video.py +++ b/tests/test_camera_video.py @@ -63,8 +63,11 @@ def test_camera_video_publishes_a_playable_mp4_per_camera_with_its_clock( } for topic in camera_topics: published_mp4 = Path(video_run.artifact_uris[video_artifact_name(topic)]) - # Published under the data root, not left in the scratch workdir. + # Published under the data root but NOT in the scratch workdir. assert published_mp4.is_relative_to(data_root) + assert "scratch" not in published_mp4.parts, ( + f"Artifact published under scratch: {published_mp4}" + ) assert published_mp4.suffix == ".mp4" labels = video_run.result.labels assert labels[f"{topic}/video_fps"] == pytest.approx(15.0, rel=0.05)