From 9c1a9fe3c2c7ca27168db3929b795b5c82a8c5f2 Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Tue, 15 Sep 2026 10:52:30 -0400
Subject: [PATCH 01/10] Skip unreferenced static files when ingesting edX
 course archives

Everything under an OLX export's static/ directory was ingested, including
assets left over from earlier offerings. u-lab carries 27 syllabus files back
to a 2014 edition, and they dominated retrieval for the questions they are
worst at answering.

documents_from_olx now skips, in addition to staff-only subtrees:

- static files no block refers to, matched on filename against the course text
  so that "asset-v1:...+type@asset+block/<name>" links count as well as
  "/static/<name>", and percent-encoded and entity-escaped spellings do too
- legacy transcripts (subs_<id>.srt.sjson) whose video id no block declares,
  read from video elements rather than by matching raw attribute text
- the asset manifests and info/updates.items.json, which list or mention every
  asset and would otherwise keep all of them alive; live announcements in that
  file still count as references, deleted ones do not

Staff-only files are excluded as reference sources too, so an asset only a
hidden block mentions is unreferenced.

unpublish_staff_only_files becomes unpublish_excluded_files and walks the same
exclusion function, so what ingestion skips and what the cleanup removes cannot
drift. It gains --dry-run and a --report CSV written by the command process,
since the tasks fan out across workers.

audit_olx_references reports the same filter against an extracted archive, so
the per-course numbers can be reproduced without S3.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 learning_resources/etl/edx_shared.py          |  60 ++++--
 learning_resources/etl/edx_shared_test.py     | 120 ++++++++---
 learning_resources/etl/utils.py               | 176 ++++++++++++++-
 learning_resources/etl/utils_test.py          | 201 +++++++++++++++++-
 .../commands/audit_olx_references.py          |  61 ++++++
 .../commands/unpublish_excluded_files.py      | 108 ++++++++++
 .../commands/unpublish_excluded_files_test.py |  93 ++++++++
 .../commands/unpublish_staff_only_files.py    |  69 ------
 learning_resources/tasks.py                   |  33 ++-
 learning_resources/tasks_test.py              |  25 ++-
 main/celery.py                                |   4 +-
 11 files changed, 815 insertions(+), 135 deletions(-)
 create mode 100644 learning_resources/management/commands/audit_olx_references.py
 create mode 100644 learning_resources/management/commands/unpublish_excluded_files.py
 create mode 100644 learning_resources/management/commands/unpublish_excluded_files_test.py
 delete mode 100644 learning_resources/management/commands/unpublish_staff_only_files.py

diff --git a/learning_resources/etl/edx_shared.py b/learning_resources/etl/edx_shared.py
index 71fff4dbf3..f9bd0aa023 100644
--- a/learning_resources/etl/edx_shared.py
+++ b/learning_resources/etl/edx_shared.py
@@ -15,10 +15,10 @@
 from learning_resources.etl.loaders import load_content_files
 from learning_resources.etl.utils import (
     calc_checksum,
+    excluded_olx_paths,
     get_bucket_by_name,
     get_edx_module_id,
     get_s3_prefix_for_source,
-    staff_only_olx_paths,
     transform_content_files,
 )
 from learning_resources.models import ContentFile, LearningResourceRun
@@ -369,20 +369,29 @@ def sync_edx_course_files(
     )
 
 
-def unpublish_staff_only_content_files(
-    etl_source: str, ids: list[int], keys: list[str]
-) -> int:
+def unpublish_excluded_content_files(
+    etl_source: str,
+    ids: list[int],
+    keys: list[str],
+    *,
+    dry_run: bool = False,
+    report: bool = False,
+) -> tuple[int, list[dict]]:
     """
-    Unpublish (and deindex) content files under staff-only OLX subtrees for the
-    runs matching the given archive keys, without re-extracting anything.
+    Unpublish (and deindex) content files the course does not use — staff-only
+    subtrees, asset manifests and unreferenced static files — for the runs
+    matching the given archive keys, without re-extracting anything.
 
     Args:
         etl_source(str): The edx ETL source
         ids(list of int): list of course ids to process
         keys(list[str]): list of S3 archive keys to search through
+        dry_run(bool): count the rows but leave them published and deindex nothing
+        report(bool): return a row per excluded file for the caller to write out
 
     Returns:
-        int: number of content files unpublished
+        tuple of (int, list of dict): number of content files unpublished (or
+            that would be, under dry_run), and the report rows when requested
     """
     from learning_resources_search import tasks as search_tasks
     from vector_search import tasks as vector_tasks
@@ -390,6 +399,7 @@ def unpublish_staff_only_content_files(
     bucket = get_bucket_by_name(settings.COURSE_ARCHIVE_BUCKET_NAME)
     run_lookup = build_run_lookup(etl_source, ids)
     total = 0
+    rows = []
     for key in keys:
         matching_runs = run_lookup.get(extract_run_id_from_key(etl_source, key))
         if not matching_runs:
@@ -408,23 +418,41 @@ def unpublish_staff_only_content_files(
             if olx_path is None:
                 continue
             try:
-                hidden_paths = staff_only_olx_paths(olx_path)
+                excluded_paths = excluded_olx_paths(olx_path)
             except ElementTree.ParseError:
                 log.exception("Malformed OLX in %s, skipping", key)
                 continue
-            hidden_keys = {get_edx_module_id(str(path), run) for path in hidden_paths}
-        if not hidden_keys:
+            excluded_keys = {
+                get_edx_module_id(str(path), run): str(path.relative_to(olx_path))
+                for path in excluded_paths
+            }
+        if not excluded_keys:
             continue
         # scoped to this run: keys embed the run_id, but never rely on that alone
-        hidden_files = ContentFile.objects.filter(run=run, key__in=hidden_keys)
-        unpublished = hidden_files.filter(published=True).update(published=False)
+        excluded_files = ContentFile.objects.filter(run=run, key__in=excluded_keys)
+        if report:
+            rows.extend(
+                {
+                    "run_id": run.run_id,
+                    "key": content_key,
+                    "source_path": excluded_keys.get(content_key, ""),
+                    "published": published,
+                }
+                for content_key, published in excluded_files.values_list(
+                    "key", "published"
+                )
+            )
+        if dry_run:
+            total += excluded_files.filter(published=True).count()
+            continue
+        unpublished = excluded_files.filter(published=True).update(published=False)
         total += unpublished
         log.info(
-            "Unpublished %d staff-only content files for %s", unpublished, run.run_id
+            "Unpublished %d excluded content files for %s", unpublished, run.run_id
         )
-        # dispatched whenever hidden rows exist, not only when this call flipped
+        # dispatched whenever excluded rows exist, not only when this call flipped
         # them, so a re-run after a failed deindex task cleans up the indexes
-        if hidden_files.exists():
+        if excluded_files.exists():
             search_tasks.deindex_run_content_files.delay(run.id, unpublished_only=True)
             vector_tasks.remove_unpublished_run_content_files.delay(run.id)
-    return total
+    return total, rows
diff --git a/learning_resources/etl/edx_shared_test.py b/learning_resources/etl/edx_shared_test.py
index 1d9cc34aa6..b5157d8ccd 100644
--- a/learning_resources/etl/edx_shared_test.py
+++ b/learning_resources/etl/edx_shared_test.py
@@ -19,7 +19,7 @@
     normalize_run_id,
     process_course_archive,
     sync_edx_course_files,
-    unpublish_staff_only_content_files,
+    unpublish_excluded_content_files,
 )
 from learning_resources.etl.utils import get_edx_module_id, get_s3_prefix_for_source
 from learning_resources.factories import (
@@ -1670,6 +1670,8 @@ def _staff_only_archive(tmp_path) -> Path:
         "vertical/v_staff.xml": '<vertical><html url_name="h_staff"/></vertical>',
         "html/h_staff.xml": '<html filename="h_staff"/>',
         "html/h_staff.html": "<p>staff</p>",
+        # nothing links this, so it is from an earlier offering
+        "static/stale_syllabus.pdf": "stale",
     }
     for rel, text in files.items():
         (olx / rel).parent.mkdir(parents=True, exist_ok=True)
@@ -1712,7 +1714,7 @@ def mock_deindex_tasks(mocker):
     )
 
 
-def test_unpublish_staff_only_content_files(staff_only_run, mock_deindex_tasks):
+def test_unpublish_excluded_content_files(staff_only_run, mock_deindex_tasks):
     """Only the matching run's staff-only content files are unpublished and deindexed"""
     run = staff_only_run.run
     other_run = LearningResourceRunFactory.create(
@@ -1728,7 +1730,7 @@ def test_unpublish_staff_only_content_files(staff_only_run, mock_deindex_tasks):
     for cf_key in run_keys.values():
         ContentFileFactory.create(run=other_run, key=cf_key, published=True)
 
-    unpublished = unpublish_staff_only_content_files(
+    unpublished, _ = unpublish_excluded_content_files(
         staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
     )
 
@@ -1744,7 +1746,7 @@ def test_unpublish_staff_only_content_files(staff_only_run, mock_deindex_tasks):
     mock_deindex_tasks.qdrant.assert_called_once_with(run.id)
 
 
-def test_unpublish_staff_only_content_files_nothing_hidden(
+def test_unpublish_excluded_content_files_nothing_hidden(
     staff_only_run, mock_deindex_tasks
 ):
     """No deindex tasks are queued when a run has no staff-only content files"""
@@ -1753,21 +1755,18 @@ def test_unpublish_staff_only_content_files_nothing_hidden(
         run=run, key=get_edx_module_id("course/html/h_ok.xml", run), published=True
     )
 
-    assert (
-        unpublish_staff_only_content_files(
-            staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
-        )
-        == 0
-    )
+    assert unpublish_excluded_content_files(
+        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
+    ) == (0, [])
     mock_deindex_tasks.opensearch.assert_not_called()
 
 
-def test_unpublish_staff_only_content_files_malformed_archive(
+def test_unpublish_excluded_content_files_malformed_archive(
     staff_only_run, mock_deindex_tasks, mocker
 ):
     """A malformed archive is skipped without touching its content files"""
     mocker.patch(
-        "learning_resources.etl.edx_shared.staff_only_olx_paths",
+        "learning_resources.etl.edx_shared.excluded_olx_paths",
         side_effect=ElementTree.ParseError("bad"),
     )
     ContentFileFactory.create(
@@ -1776,17 +1775,14 @@ def test_unpublish_staff_only_content_files_malformed_archive(
         published=True,
     )
 
-    assert (
-        unpublish_staff_only_content_files(
-            staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
-        )
-        == 0
-    )
+    assert unpublish_excluded_content_files(
+        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
+    ) == (0, [])
     assert ContentFile.objects.filter(run=staff_only_run.run, published=True).exists()
     mock_deindex_tasks.opensearch.assert_not_called()
 
 
-def test_unpublish_staff_only_content_files_rerun_redeindexes(
+def test_unpublish_excluded_content_files_rerun_redeindexes(
     staff_only_run, mock_deindex_tasks
 ):
     """A re-run with already-unpublished hidden files still queues the deindex tasks"""
@@ -1795,11 +1791,85 @@ def test_unpublish_staff_only_content_files_rerun_redeindexes(
         run=run, key=get_edx_module_id("course/html/h_staff.xml", run), published=False
     )
 
-    assert (
-        unpublish_staff_only_content_files(
-            staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
-        )
-        == 0
-    )
+    assert unpublish_excluded_content_files(
+        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
+    ) == (0, [])
     mock_deindex_tasks.opensearch.assert_called_once_with(run.id, unpublished_only=True)
     mock_deindex_tasks.qdrant.assert_called_once_with(run.id)
+
+
+def test_unpublish_excluded_content_files_drops_unreferenced_static(
+    staff_only_run, mock_deindex_tasks
+):
+    """A static file no block refers to is unpublished alongside staff-only files"""
+    run = staff_only_run.run
+    stale_key = get_edx_module_id("course/static/stale_syllabus.pdf", run)
+    ContentFileFactory.create(run=run, key=stale_key, published=True)
+
+    unpublished, _ = unpublish_excluded_content_files(
+        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
+    )
+
+    assert unpublished == 1
+    assert not ContentFile.objects.filter(
+        run=run, key=stale_key, published=True
+    ).exists()
+
+
+def test_unpublish_excluded_content_files_dry_run(staff_only_run, mock_deindex_tasks):
+    """A dry run counts the rows but changes nothing and deindexes nothing"""
+    run = staff_only_run.run
+    ContentFileFactory.create(
+        run=run,
+        key=get_edx_module_id("course/static/stale_syllabus.pdf", run),
+        published=True,
+    )
+
+    unpublished, _ = unpublish_excluded_content_files(
+        staff_only_run.source,
+        [staff_only_run.course.id],
+        [staff_only_run.key],
+        dry_run=True,
+    )
+
+    assert unpublished == 1
+    assert ContentFile.objects.filter(run=run, published=True).count() == 1
+    mock_deindex_tasks.opensearch.assert_not_called()
+    mock_deindex_tasks.qdrant.assert_not_called()
+
+
+def test_unpublish_excluded_content_files_report_rows(
+    staff_only_run, mock_deindex_tasks
+):
+    """Report rows name the run, key and source path of every excluded file"""
+    run = staff_only_run.run
+    stale_key = get_edx_module_id("course/static/stale_syllabus.pdf", run)
+    ContentFileFactory.create(run=run, key=stale_key, published=True)
+
+    _, rows = unpublish_excluded_content_files(
+        staff_only_run.source,
+        [staff_only_run.course.id],
+        [staff_only_run.key],
+        dry_run=True,
+        report=True,
+    )
+
+    assert [row["key"] for row in rows] == [stale_key]
+    assert rows[0]["run_id"] == run.run_id
+    assert rows[0]["source_path"] == "static/stale_syllabus.pdf"
+    assert rows[0]["published"] is True
+
+
+def test_unpublish_excluded_content_files_no_report_by_default(
+    staff_only_run, mock_deindex_tasks
+):
+    """Rows are only collected when asked for, since they cross the result backend"""
+    ContentFileFactory.create(
+        run=staff_only_run.run,
+        key=get_edx_module_id("course/static/stale_syllabus.pdf", staff_only_run.run),
+        published=True,
+    )
+    _, rows = unpublish_excluded_content_files(
+        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
+    )
+    assert rows == []
diff --git a/learning_resources/etl/utils.py b/learning_resources/etl/utils.py
index bd2a76eb41..a97f12be68 100644
--- a/learning_resources/etl/utils.py
+++ b/learning_resources/etl/utils.py
@@ -2,6 +2,7 @@
 
 import base64
 import glob
+import html
 import json
 import logging
 import math
@@ -18,6 +19,7 @@
 from io import BytesIO
 from pathlib import Path
 from tempfile import TemporaryDirectory
+from urllib.parse import unquote
 
 import boto3
 import pypdfium2 as pdfium
@@ -380,11 +382,179 @@ def staff_only_olx_paths(olx_path: str | Path) -> set[Path]:
     return hidden
 
 
+REFERENCE_SCAN_EXTENSIONS = frozenset({".xml", ".html", ".htm", ".json", ".txt", ".md"})
+
+# Asset manifests list every file in the export and updates.items.json is mostly
+# an archive of deleted announcements. None of them describes current course
+# content, and treating them as references keeps every stale asset alive.
+NON_CONTENT_OLX_FILES = (
+    "policies/assets.json",
+    "assets/assets.xml",
+    "info/updates.items.json",
+)
+COURSE_UPDATES_FILE = "info/updates.items.json"
+
+# A legacy transcript is named for its video's id rather than for anything the
+# course text contains, so the id is the only link back to the block using it.
+VIDEO_ID_ATTRIBUTES = ("sub", "youtube", "youtube_id_1_0")
+LEGACY_TRANSCRIPT_RE = re.compile(
+    r"^(?:[a-z]{2}(?:[-_][a-z]{2})?_)?subs_(.+)\.srt\.sjson$", re.IGNORECASE
+)
+
+
+def normalize_asset_ref(text: str) -> str:
+    """Collapse the spellings a filename takes on disk vs. in a reference"""
+    return re.sub(r"[\s_]", "", unquote(html.unescape(text)).lower())
+
+
+def _olx_reference_sources(root: Path, skip: set[Path]) -> list[Path]:
+    """Files whose text may legitimately refer to a static asset"""
+    sources = []
+    for path in root.rglob("*"):
+        if not path.is_file() or path.suffix.lower() not in REFERENCE_SCAN_EXTENSIONS:
+            continue
+        relative = path.relative_to(root)
+        if (
+            relative.parts[0] == "static"
+            or relative.as_posix() in NON_CONTENT_OLX_FILES
+            or path in skip
+            or any("draft" in part for part in relative.parts[:-1])
+        ):
+            continue
+        sources.append(path)
+    return sources
+
+
+def _live_course_updates(root: Path) -> str:
+    """
+    Text of the announcements the course team has not deleted. The whole file is
+    excluded as a reference source because edX keeps deleted announcements in
+    the export, but a live one still counts.
+    """
+    try:
+        items = json.loads((root / COURSE_UPDATES_FILE).read_text(errors="ignore"))
+    except (OSError, ValueError):
+        return ""
+    return "\n".join(
+        item.get("content") or ""
+        for item in items
+        if isinstance(item, dict) and item.get("status") != "deleted"
+    )
+
+
+def _olx_video_ids(sources: list[Path]) -> set[str] | None:
+    """
+    Video ids declared anywhere in the course, or None if any source could not
+    be parsed. None means "ids unknown", and callers keep every legacy
+    transcript rather than drop one whose video they failed to read.
+    """
+    ids = set()
+    for path in sources:
+        if path.suffix.lower() != ".xml":
+            continue
+        try:
+            element = ElementTree.parse(path).getroot()
+        except ElementTree.ParseError:
+            log.warning("Malformed XML in %s, keeping all legacy transcripts", path)
+            return None
+        # iter() finds <video> whether it has its own file or sits inline
+        for video in element.iter("video"):
+            for attribute in VIDEO_ID_ATTRIBUTES:
+                # youtube is "<speed>:<id>", the others are bare ids
+                value = (video.get(attribute) or "").split(":")[-1]
+                if value:
+                    ids.add(normalize_asset_ref(value))
+    return ids
+
+
+def static_olx_references(root: Path, skip: set[Path]) -> dict[Path, str | None]:
+    """
+    Map every file under static/ to the source file that refers to it, or None
+    when nothing in the course does.
+
+    Matching is on the filename rather than on a "/static/" prefix, because
+    courses also link assets as "asset-v1:...+type@asset+block/<name>", and it
+    is a substring test rather than a parse so that an unanticipated spelling
+    keeps a file rather than dropping it.
+
+    Args:
+        root (Path): the root of the OLX tree
+        skip (set[Path]): files that must not count as references, i.e. the
+            staff-only set, so an asset only an answer key mentions is unreferenced
+
+    Returns:
+        dict of Path to str or None: static file -> relative path of its referrer
+    """
+    static_dir = root / "static"
+    if not static_dir.is_dir():
+        return {}
+    sources = _olx_reference_sources(root, skip)
+    texts = [
+        (
+            path.relative_to(root).as_posix(),
+            normalize_asset_ref(path.read_text(errors="ignore")),
+        )
+        for path in sources
+    ]
+    updates = normalize_asset_ref(_live_course_updates(root))
+    if updates:
+        texts.append((COURSE_UPDATES_FILE, updates))
+    blob = "\n".join(text for _, text in texts)
+    video_ids = _olx_video_ids(sources)
+
+    references = {}
+    for path in sorted(static_dir.rglob("*")):
+        if not path.is_file():
+            continue
+        name = normalize_asset_ref(path.name)
+        if name in blob:
+            references[path] = next(
+                (source for source, text in texts if name in text), None
+            )
+            continue
+        legacy = LEGACY_TRANSCRIPT_RE.match(path.name)
+        if legacy and (
+            video_ids is None or normalize_asset_ref(legacy.group(1)) in video_ids
+        ):
+            references[path] = "video block id"
+            continue
+        references[path] = None
+    return references
+
+
+def excluded_olx_paths(olx_path: str | Path) -> set[Path]:
+    """
+    Files an OLX export contains that the course itself does not use: staff-only
+    subtrees, the asset manifests and announcement archive, and anything under
+    static/ that nothing refers to. See hq#13350.
+
+    Args:
+        olx_path (str or Path): The path to the directory with the OLX data
+
+    Returns:
+        set of Path: files that should not be ingested
+    """
+    root = Path(olx_path)
+    excluded = staff_only_olx_paths(root)
+    if not (root / "course.xml").is_file():
+        # not an OLX export; Canvas archives reach here via process_olx_path
+        return excluded
+    excluded.update(
+        root / name for name in NON_CONTENT_OLX_FILES if (root / name).is_file()
+    )
+    excluded.update(
+        path
+        for path, referrer in static_olx_references(root, excluded).items()
+        if referrer is None
+    )
+    return excluded
+
+
 def documents_from_olx(
     olx_path: str, valid_file_types: list[str] = VALID_TEXT_FILE_TYPES
 ) -> Generator[tuple, None, None]:
     """
-    Extract text from OLX directory, skipping staff-only content
+    Extract text from OLX directory, skipping content the course does not use
 
     Args:
         olx_path (str): The path to the directory with the OLX data
@@ -392,13 +562,13 @@ def documents_from_olx(
     Yields:
         tuple: A list of (bytes of content, metadata)
     """
-    staff_only = staff_only_olx_paths(olx_path)
+    excluded = excluded_olx_paths(olx_path)
     for root, _, files in os.walk(olx_path):
         path = "/".join(root.split("/")[3:])
         for filename in files:
             extension_lower = Path(filename).suffix.lower()
 
-            if Path(root, filename) in staff_only:
+            if Path(root, filename) in excluded:
                 continue
             if extension_lower in valid_file_types and "draft" not in root:
                 with Path.open(Path(root, filename), "rb") as f:
diff --git a/learning_resources/etl/utils_test.py b/learning_resources/etl/utils_test.py
index 3b68b15abf..5ec25af549 100644
--- a/learning_resources/etl/utils_test.py
+++ b/learning_resources/etl/utils_test.py
@@ -282,7 +282,13 @@ def test_transform_content_files(  # noqa: PLR0913
 def test_documents_from_olx():
     """Test for documents_from_olx"""
     parsed_documents = get_olx_test_docs()
-    assert len(parsed_documents) == 92
+    # the archive's two asset manifests are excluded, everything else is yielded
+    assert len(parsed_documents) == 90
+    assert not [
+        doc
+        for doc in parsed_documents
+        if doc[1]["source_path"].endswith(("policies/assets.json", "assets/assets.xml"))
+    ]
 
     formula2do = next(
         doc
@@ -318,9 +324,18 @@ def test_documents_from_olx_skips_staff_only_subtrees(tmp_path):
         "sequential/seq_ok.xml",
         '<sequential><vertical url_name="v_ok"/><vertical url_name="v_staff"/></sequential>',
     )
-    _write_olx(olx, "vertical/v_ok.xml", '<vertical><html url_name="h_ok"/></vertical>')
+    _write_olx(
+        olx,
+        "vertical/v_ok.xml",
+        '<vertical><html url_name="h_ok"/><video url_name="vid_ok"/></vertical>',
+    )
     _write_olx(olx, "html/h_ok.xml", '<html filename="h_ok"/>')
     _write_olx(olx, "html/h_ok.html", "<p>visible</p>")
+    _write_olx(
+        olx,
+        "video/vid_ok.xml",
+        '<video url_name="vid_ok"><transcript language="en" src="visible.srt"/></video>',
+    )
     _write_olx(
         olx,
         "vertical/v_staff.xml",
@@ -368,6 +383,7 @@ def test_documents_from_olx_skips_staff_only_subtrees(tmp_path):
         "static/visible.srt",
         "tabs/syllabus.html",
         "vertical/v_ok.xml",
+        "video/vid_ok.xml",
     ]
 
 
@@ -415,6 +431,187 @@ def test_documents_from_olx_malformed_block_fails_closed(tmp_path, bad_file):
         list(utils.documents_from_olx(str(olx)))
 
 
+def _reference_olx(tmp_path, **static_files):
+    """Minimal OLX course with one visible html block and the given static files"""
+    olx = tmp_path / "course"
+    _write_olx(olx, "course.xml", '<course url_name="run" org="MITx" course="1"/>')
+    _write_olx(olx, "course/run.xml", '<course><chapter url_name="ch"/></course>')
+    _write_olx(olx, "chapter/ch.xml", '<chapter><vertical url_name="v"/></chapter>')
+    _write_olx(olx, "vertical/v.xml", '<vertical><html url_name="h"/></vertical>')
+    _write_olx(olx, "html/h.xml", '<html filename="h"/>')
+    for name, text in static_files.items():
+        _write_olx(olx, f"static/{name}", text)
+    return olx
+
+
+def _olx_source_paths(olx):
+    prefix = "/".join(str(olx).split("/")[3:]) + "/"
+    return sorted(
+        meta["source_path"].removeprefix(prefix)
+        for _, meta in utils.documents_from_olx(str(olx))
+    )
+
+
+@pytest.mark.parametrize(
+    ("reference", "kept"),
+    [
+        ('<a href="/static/notes.pdf">notes</a>', True),
+        # courses also link assets by their asset key rather than /static/
+        ('<a href="/asset-v1:MITx+1+run+type@asset+block/notes.pdf">n</a>', True),
+        # percent-encoded and entity-escaped spellings are still references
+        ('<a href="/static/notes%2Epdf">notes</a>', True),
+        ('<a href="/static/notes.pdf?raw=1&amp;v=2">notes</a>', True),
+        ("<p>nothing here</p>", False),
+    ],
+)
+def test_documents_from_olx_static_reference_spellings(tmp_path, reference, kept):
+    """A static file survives when any block spells its name, however it spells it"""
+    olx = _reference_olx(tmp_path, **{"notes.pdf": "pdf bytes"})
+    _write_olx(olx, "html/h.html", reference)
+    assert ("static/notes.pdf" in _olx_source_paths(olx)) is kept
+
+
+def test_documents_from_olx_normalizes_spaces_and_underscores(tmp_path):
+    """A file stored with spaces matches a reference written with underscores"""
+    olx = _reference_olx(tmp_path, **{"Course Syllabus V2.pdf": "pdf"})
+    _write_olx(olx, "html/h.html", '<a href="/static/Course_Syllabus_V2.pdf">s</a>')
+    assert "static/Course Syllabus V2.pdf" in _olx_source_paths(olx)
+
+
+def test_documents_from_olx_skips_unreferenced_static_files(tmp_path):
+    """Static files no block refers to are not ingested (hq#13350)"""
+    olx = _reference_olx(
+        tmp_path, **{"current.pdf": "current", "syllabus_2015.pdf": "stale"}
+    )
+    _write_olx(olx, "html/h.html", '<a href="/static/current.pdf">current</a>')
+    paths = _olx_source_paths(olx)
+    assert "static/current.pdf" in paths
+    assert "static/syllabus_2015.pdf" not in paths
+
+
+def test_documents_from_olx_ignores_asset_manifests(tmp_path):
+    """Manifests list every asset, so they cannot count as references"""
+    olx = _reference_olx(tmp_path, **{"stale.pdf": "stale"})
+    _write_olx(olx, "html/h.html", "<p>no links</p>")
+    _write_olx(olx, "policies/assets.json", '{"stale.pdf": {"filename": "stale.pdf"}}')
+    _write_olx(olx, "assets/assets.xml", "<assets><asset>stale.pdf</asset></assets>")
+    paths = _olx_source_paths(olx)
+    assert "static/stale.pdf" not in paths
+    assert "policies/assets.json" not in paths
+    assert "assets/assets.xml" not in paths
+
+
+def test_documents_from_olx_ignores_deleted_announcements(tmp_path):
+    """Deleted announcements are archived by edX but no longer part of the course"""
+    olx = _reference_olx(tmp_path, **{"old.pdf": "old", "new.pdf": "new"})
+    _write_olx(olx, "html/h.html", "<p>no links</p>")
+    _write_olx(
+        olx,
+        "info/updates.items.json",
+        '[{"id": 1, "status": "deleted", "content": "see /static/old.pdf"},'
+        ' {"id": 2, "status": "visible", "content": "see /static/new.pdf"}]',
+    )
+    paths = _olx_source_paths(olx)
+    assert "static/old.pdf" not in paths
+    assert "static/new.pdf" in paths
+    # the file itself is a decade of announcements, not current course content
+    assert "info/updates.items.json" not in paths
+
+
+def test_documents_from_olx_drops_assets_only_staff_blocks_mention(tmp_path):
+    """An answer key a hidden block links is unreferenced, not referenced"""
+    olx = tmp_path / "course"
+    _write_olx(olx, "course.xml", '<course url_name="run" org="MITx" course="1"/>')
+    _write_olx(
+        olx,
+        "course/run.xml",
+        '<course><chapter url_name="ch_ok"/><chapter url_name="ch_staff"/></course>',
+    )
+    _write_olx(olx, "chapter/ch_ok.xml", '<chapter><vertical url_name="v"/></chapter>')
+    _write_olx(olx, "vertical/v.xml", '<vertical><html url_name="h"/></vertical>')
+    _write_olx(olx, "html/h.xml", '<html filename="h"/>')
+    _write_olx(olx, "html/h.html", "<p>visible</p>")
+    _write_olx(
+        olx,
+        "chapter/ch_staff.xml",
+        '<chapter visible_to_staff_only="true">'
+        '<vertical url_name="v_staff"/></chapter>',
+    )
+    _write_olx(
+        olx, "vertical/v_staff.xml", '<vertical><html url_name="h_staff"/></vertical>'
+    )
+    _write_olx(olx, "html/h_staff.xml", '<html filename="h_staff"/>')
+    _write_olx(olx, "html/h_staff.html", '<a href="/static/answers.pdf">key</a>')
+    _write_olx(olx, "static/answers.pdf", "answers")
+    assert "static/answers.pdf" not in _olx_source_paths(olx)
+
+
+def test_documents_from_olx_drafts_do_not_keep_assets(tmp_path):
+    """Drafts are not ingested, so their links do not keep an asset alive"""
+    olx = _reference_olx(tmp_path, **{"draft_only.pdf": "pdf"})
+    _write_olx(olx, "html/h.html", "<p>no links</p>")
+    _write_olx(olx, "drafts/html/d.html", '<a href="/static/draft_only.pdf">d</a>')
+    assert "static/draft_only.pdf" not in _olx_source_paths(olx)
+
+
+@pytest.mark.parametrize(
+    "video_xml",
+    [
+        # every spelling of the attribute is valid XML and must be honoured
+        '<video url_name="vid" sub="AbC123"/>',
+        "<video url_name='vid' sub='AbC123'/>",
+        '<video url_name="vid" sub = "AbC123"/>',
+        '<video url_name="vid" youtube="1.00:AbC123"/>',
+        '<video url_name="vid" youtube_id_1_0="AbC123"/>',
+    ],
+)
+def test_documents_from_olx_keeps_legacy_transcripts(tmp_path, video_xml):
+    """subs_<id>.srt.sjson is named for the video id, never for its own filename"""
+    olx = _reference_olx(
+        tmp_path,
+        **{"subs_AbC123.srt.sjson": "{}", "es_subs_AbC123.srt.sjson": "{}"},
+    )
+    _write_olx(olx, "html/h.html", "<p>no links</p>")
+    _write_olx(olx, "vertical/v.xml", '<vertical><video url_name="vid"/></vertical>')
+    _write_olx(olx, "video/vid.xml", video_xml)
+    paths = _olx_source_paths(olx)
+    assert "static/subs_AbC123.srt.sjson" in paths
+    assert "static/es_subs_AbC123.srt.sjson" in paths
+
+
+def test_documents_from_olx_drops_orphaned_legacy_transcripts(tmp_path):
+    """A legacy transcript whose video is gone belongs to an earlier offering"""
+    olx = _reference_olx(
+        tmp_path, **{"subs_Current.srt.sjson": "{}", "subs_Removed.srt.sjson": "{}"}
+    )
+    _write_olx(olx, "html/h.html", "<p>no links</p>")
+    _write_olx(olx, "vertical/v.xml", '<vertical><video url_name="vid"/></vertical>')
+    _write_olx(olx, "video/vid.xml", '<video url_name="vid" sub="Current"/>')
+    paths = _olx_source_paths(olx)
+    assert "static/subs_Current.srt.sjson" in paths
+    assert "static/subs_Removed.srt.sjson" not in paths
+
+
+def test_documents_from_olx_keeps_inline_video_transcripts(tmp_path):
+    """A <video> inline in a parent declares its transcripts just as a file does"""
+    olx = _reference_olx(tmp_path, **{"subs_Inline.srt.sjson": "{}"})
+    _write_olx(olx, "html/h.html", "<p>no links</p>")
+    _write_olx(
+        olx,
+        "vertical/v.xml",
+        '<vertical><video url_name="vid" sub="Inline"/></vertical>',
+    )
+    assert "static/subs_Inline.srt.sjson" in _olx_source_paths(olx)
+
+
+def test_documents_from_olx_unparsable_xml_keeps_legacy_transcripts(tmp_path):
+    """Unknown video ids must not be read as "no video uses this transcript\""""
+    olx = _reference_olx(tmp_path, **{"subs_AbC123.srt.sjson": "{}"})
+    _write_olx(olx, "html/h.html", "<p>no links</p>")
+    _write_olx(olx, "tabs/broken.xml", "<tab")
+    assert "static/subs_AbC123.srt.sjson" in _olx_source_paths(olx)
+
+
 def test_documents_from_olx_without_course_xml_yields_everything(tmp_path):
     """Non-OLX trees (canvas, tutor problem sets) are not filtered"""
     olx = tmp_path / "course"
diff --git a/learning_resources/management/commands/audit_olx_references.py b/learning_resources/management/commands/audit_olx_references.py
new file mode 100644
index 0000000000..c75da40f69
--- /dev/null
+++ b/learning_resources/management/commands/audit_olx_references.py
@@ -0,0 +1,61 @@
+"""Report what excluded_olx_paths would drop from an extracted OLX course tree"""
+
+from collections import Counter
+from pathlib import Path
+
+from django.core.management import BaseCommand
+
+from learning_resources.constants import VALID_TEXT_FILE_TYPES
+from learning_resources.etl.utils import staff_only_olx_paths, static_olx_references
+
+TRANSCRIPT_EXTENSIONS = (".srt", ".sjson", ".vtt")
+
+
+class Command(BaseCommand):
+    """
+    Check the ingestion filter against a real archive before pointing
+    unpublish_excluded_files at production. Takes extracted OLX trees, e.g.
+
+        docker compose run --rm -v /path/to/extracted:/archives web \\
+            ./manage.py audit_olx_references /archives/<course>
+    """
+
+    help = "Report the static files an OLX course tree does not reference"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "olx_paths", nargs="+", help="Paths to extracted OLX course directories"
+        )
+
+    def handle(self, *args, **options):  # noqa: ARG002
+        """Print per-archive drop counts split by file class"""
+        for olx_path in options["olx_paths"]:
+            root = Path(olx_path)
+            references = {
+                path: referrer
+                for path, referrer in static_olx_references(
+                    root, staff_only_olx_paths(root)
+                ).items()
+                if path.suffix.lower() in VALID_TEXT_FILE_TYPES
+            }
+            dropped = [
+                path for path, referrer in references.items() if referrer is None
+            ]
+            transcripts = [
+                path for path in dropped if path.suffix.lower() in TRANSCRIPT_EXTENSIONS
+            ]
+            documents = [path for path in dropped if path not in set(transcripts)]
+            percent = 100 * len(dropped) / len(references) if references else 0
+            types = ", ".join(
+                f"{extension}:{count}"
+                for extension, count in Counter(
+                    path.suffix.lower() for path in documents
+                ).most_common(4)
+            )
+            self.stdout.write(f"\n=== {root}")
+            self.stdout.write(f"  ingestable static files : {len(references)}")
+            self.stdout.write(
+                f"  dropped                 : {len(dropped)} ({percent:.0f}%)"
+            )
+            self.stdout.write(f"    transcripts           : {len(transcripts)}")
+            self.stdout.write(f"    documents             : {len(documents)}  {types}")
diff --git a/learning_resources/management/commands/unpublish_excluded_files.py b/learning_resources/management/commands/unpublish_excluded_files.py
new file mode 100644
index 0000000000..9a19411d0e
--- /dev/null
+++ b/learning_resources/management/commands/unpublish_excluded_files.py
@@ -0,0 +1,108 @@
+"""Unpublish edX content files that the course itself does not use"""
+
+import csv
+from pathlib import Path
+
+from django.core.management import BaseCommand, CommandError
+
+from learning_resources.etl.constants import ETLSource
+from learning_resources.tasks import unpublish_all_excluded_files
+from main import settings
+from main.utils import now_in_utc
+
+EDX_SOURCES = [
+    ETLSource.mitxonline.name,
+    ETLSource.mit_edx.name,
+    ETLSource.xpro.name,
+    ETLSource.oll.name,
+]
+REPORT_FIELDS = ("etl_source", "run_id", "key", "source_path", "published")
+
+
+class Command(BaseCommand):
+    """
+    Walk each course's current archive and unpublish the content files it
+    excludes — staff-only subtrees, asset manifests, and static files no block
+    refers to — then deindex them. Nothing is re-extracted or re-embedded.
+    """
+
+    help = "Unpublish unused edX content files from existing archives"
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--source",
+            dest="sources",
+            action="append",
+            choices=EDX_SOURCES,
+            help="ETL source to process (repeatable). Default: all edX sources",
+        )
+        parser.add_argument(
+            "-c",
+            "--chunk-size",
+            dest="chunk_size",
+            default=settings.LEARNING_COURSE_ITERATOR_CHUNK_SIZE,
+            type=int,
+            help="Chunk size for batch task",
+        )
+        parser.add_argument(
+            "--resource-ids",
+            dest="learning_resource_ids",
+            required=False,
+            help="If set, only process the learning resources with these ids",
+        )
+        parser.add_argument(
+            "--dry-run",
+            dest="dry_run",
+            action="store_true",
+            help="Report what would be unpublished without changing anything",
+        )
+        parser.add_argument(
+            "--report",
+            dest="report",
+            required=False,
+            help=(
+                "Write a CSV of every excluded content file to this path. "
+                "Requires --resource-ids, since the rows travel back through "
+                "the celery result backend"
+            ),
+        )
+
+    def handle(self, *args, **options):  # noqa: ARG002
+        """Run the unpublish tasks"""
+        resource_ids = (
+            options["learning_resource_ids"].split(",")
+            if options["learning_resource_ids"]
+            else None
+        )
+        report_path = options["report"]
+        if report_path and not resource_ids:
+            msg = "--report requires --resource-ids"
+            raise CommandError(msg)
+
+        start = now_in_utc()
+        rows = []
+        for source in options["sources"] or EDX_SOURCES:
+            task = unpublish_all_excluded_files.delay(
+                etl_source=source,
+                chunk_size=options["chunk_size"],
+                learning_resource_ids=resource_ids,
+                dry_run=options["dry_run"],
+                report=bool(report_path),
+            )
+            self.stdout.write(f"Started task {task} for {source}, waiting...")
+            total = 0
+            for count, source_rows in task.get() or []:
+                total += count or 0
+                rows.extend({"etl_source": source, **row} for row in source_rows or [])
+            verb = "would unpublish" if options["dry_run"] else "unpublished"
+            self.stdout.write(f"{source}: {verb} {total} content files")
+
+        if report_path:
+            with Path(report_path).open("w", newline="") as report_file:
+                writer = csv.DictWriter(report_file, fieldnames=REPORT_FIELDS)
+                writer.writeheader()
+                writer.writerows(rows)
+            self.stdout.write(f"Wrote {len(rows)} rows to {report_path}")
+
+        total_seconds = (now_in_utc() - start).total_seconds()
+        self.stdout.write(f"Finished in {total_seconds} seconds")
diff --git a/learning_resources/management/commands/unpublish_excluded_files_test.py b/learning_resources/management/commands/unpublish_excluded_files_test.py
new file mode 100644
index 0000000000..41b91ea29d
--- /dev/null
+++ b/learning_resources/management/commands/unpublish_excluded_files_test.py
@@ -0,0 +1,93 @@
+"""Tests for the unpublish_excluded_files management command"""
+
+import csv
+from io import StringIO
+from pathlib import Path
+
+import pytest
+from django.core.management import CommandError, call_command
+
+pytestmark = pytest.mark.django_db
+
+COMMAND = "unpublish_excluded_files"
+TASK_PATH = (
+    "learning_resources.management.commands."
+    "unpublish_excluded_files.unpublish_all_excluded_files"
+)
+
+
+@pytest.fixture
+def mock_task(mocker):
+    """Mock the fan-out task, returning one chunk result per call"""
+    task = mocker.patch(TASK_PATH)
+    task.delay.return_value.get.return_value = [(2, []), (1, [])]
+    return task
+
+
+def test_unpublish_excluded_files_runs_every_source(mock_task):
+    """Without --source the command processes all four edX sources"""
+    stdout = StringIO()
+    call_command(COMMAND, stdout=stdout)
+
+    sources = [call.kwargs["etl_source"] for call in mock_task.delay.call_args_list]
+    assert sources == ["mitxonline", "mit_edx", "xpro", "oll"]
+    assert "mitxonline: unpublished 3 content files" in stdout.getvalue()
+
+
+def test_unpublish_excluded_files_dry_run(mock_task):
+    """--dry-run is passed to the task and reported as a would-be change"""
+    stdout = StringIO()
+    call_command(COMMAND, "--source", "xpro", "--dry-run", stdout=stdout)
+
+    assert mock_task.delay.call_args.kwargs["dry_run"] is True
+    assert "xpro: would unpublish 3 content files" in stdout.getvalue()
+
+
+def test_unpublish_excluded_files_report_requires_resource_ids(mock_task):
+    """A report would have to travel back through the result backend unbounded"""
+    with pytest.raises(CommandError, match="requires --resource-ids"):
+        call_command(COMMAND, "--report", "out.csv", stdout=StringIO())
+    mock_task.delay.assert_not_called()
+
+
+def test_unpublish_excluded_files_writes_report(mock_task, tmp_path):
+    """The command process writes the CSV, since the tasks run on other workers"""
+    mock_task.delay.return_value.get.return_value = [
+        (
+            1,
+            [
+                {
+                    "run_id": "course-v1:MITx+1+run",
+                    "key": "asset-v1:MITx+1+run+type@asset+block@old.pdf",
+                    "source_path": "static/old.pdf",
+                    "published": True,
+                }
+            ],
+        )
+    ]
+    report = tmp_path / "report.csv"
+    stdout = StringIO()
+    call_command(
+        COMMAND,
+        "--source",
+        "oll",
+        "--resource-ids",
+        "1,2",
+        "--report",
+        str(report),
+        stdout=stdout,
+    )
+
+    assert mock_task.delay.call_args.kwargs["report"] is True
+    with Path(report).open() as report_file:
+        rows = list(csv.DictReader(report_file))
+    assert rows == [
+        {
+            "etl_source": "oll",
+            "run_id": "course-v1:MITx+1+run",
+            "key": "asset-v1:MITx+1+run+type@asset+block@old.pdf",
+            "source_path": "static/old.pdf",
+            "published": "True",
+        }
+    ]
+    assert "Wrote 1 rows" in stdout.getvalue()
diff --git a/learning_resources/management/commands/unpublish_staff_only_files.py b/learning_resources/management/commands/unpublish_staff_only_files.py
deleted file mode 100644
index a03e57d2c8..0000000000
--- a/learning_resources/management/commands/unpublish_staff_only_files.py
+++ /dev/null
@@ -1,69 +0,0 @@
-"""Unpublish edX content files that sit under staff-only OLX subtrees"""
-
-from django.core.management import BaseCommand
-
-from learning_resources.etl.constants import ETLSource
-from learning_resources.tasks import unpublish_all_staff_only_files
-from main import settings
-from main.utils import now_in_utc
-
-EDX_SOURCES = [
-    ETLSource.mitxonline.name,
-    ETLSource.mit_edx.name,
-    ETLSource.xpro.name,
-    ETLSource.oll.name,
-]
-
-
-class Command(BaseCommand):
-    """
-    Walk each course's current archive and unpublish content files under
-    visible_to_staff_only subtrees, then deindex them. Nothing is re-extracted
-    or re-embedded.
-    """
-
-    help = "Unpublish staff-only edX content files from existing archives"
-
-    def add_arguments(self, parser):
-        parser.add_argument(
-            "--source",
-            dest="sources",
-            action="append",
-            choices=EDX_SOURCES,
-            help="ETL source to process (repeatable). Default: all edX sources",
-        )
-        parser.add_argument(
-            "-c",
-            "--chunk-size",
-            dest="chunk_size",
-            default=settings.LEARNING_COURSE_ITERATOR_CHUNK_SIZE,
-            type=int,
-            help="Chunk size for batch task",
-        )
-        parser.add_argument(
-            "--resource-ids",
-            dest="learning_resource_ids",
-            required=False,
-            help="If set, only process the learning resources with these ids",
-        )
-
-    def handle(self, *args, **options):  # noqa: ARG002
-        """Run the unpublish tasks"""
-        resource_ids = (
-            options["learning_resource_ids"].split(",")
-            if options["learning_resource_ids"]
-            else None
-        )
-        start = now_in_utc()
-        for source in options["sources"] or EDX_SOURCES:
-            task = unpublish_all_staff_only_files.delay(
-                etl_source=source,
-                chunk_size=options["chunk_size"],
-                learning_resource_ids=resource_ids,
-            )
-            self.stdout.write(f"Started task {task} for {source}, waiting...")
-            results = task.get()
-            total = sum(count or 0 for count in results or [])
-            self.stdout.write(f"{source}: unpublished {total} content files")
-        total_seconds = (now_in_utc() - start).total_seconds()
-        self.stdout.write(f"Finished in {total_seconds} seconds")
diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py
index 7cc3a6df4f..cdb7711caf 100644
--- a/learning_resources/tasks.py
+++ b/learning_resources/tasks.py
@@ -29,7 +29,7 @@
     get_most_recent_course_archives,
     sync_edx_archive,
     sync_edx_course_files,
-    unpublish_staff_only_content_files,
+    unpublish_excluded_content_files,
 )
 from learning_resources.etl.loaders import (
     load_learning_materials,
@@ -232,23 +232,40 @@ def _content_file_resource_ids(etl_source: str, learning_resource_ids):
 
 
 @app.task(acks_late=True, reject_on_worker_lost=True)
-def unpublish_staff_only_files(ids: list[int], etl_source: str, keys: list[str]):
-    """Unpublish staff-only content files for a chunk of courses"""
-    return unpublish_staff_only_content_files(etl_source, ids, keys)
+def unpublish_excluded_files(
+    ids: list[int],
+    etl_source: str,
+    keys: list[str],
+    *,
+    dry_run: bool = False,
+    report: bool = False,
+):
+    """Unpublish unused content files for a chunk of courses"""
+    return unpublish_excluded_content_files(
+        etl_source, ids, keys, dry_run=dry_run, report=report
+    )
 
 
 @app.task(bind=True)
-def unpublish_all_staff_only_files(
-    self, *, etl_source, chunk_size=None, learning_resource_ids=None
+def unpublish_all_excluded_files(  # noqa: PLR0913
+    self,
+    *,
+    etl_source,
+    chunk_size=None,
+    learning_resource_ids=None,
+    dry_run=False,
+    report=False,
 ):
-    """Fan out unpublish_staff_only_files over an edX source's current archives"""
+    """Fan out unpublish_excluded_files over an edX source's current archives"""
     if chunk_size is None:
         chunk_size = settings.LEARNING_COURSE_ITERATOR_CHUNK_SIZE
     archive_keys = get_most_recent_course_archives(etl_source)
     return self.replace(
         celery.group(
             [
-                unpublish_staff_only_files.si(ids, etl_source, archive_keys)
+                unpublish_excluded_files.si(
+                    ids, etl_source, archive_keys, dry_run=dry_run, report=report
+                )
                 for ids in chunks(
                     _content_file_resource_ids(etl_source, learning_resource_ids),
                     chunk_size=chunk_size,
diff --git a/learning_resources/tasks_test.py b/learning_resources/tasks_test.py
index 5ac6ca66ad..2fded2555a 100644
--- a/learning_resources/tasks_test.py
+++ b/learning_resources/tasks_test.py
@@ -1554,11 +1554,11 @@ def test_get_podcast_transcripts(mocker):
 
 
 @mock_aws
-def test_unpublish_all_staff_only_files(
+def test_unpublish_all_excluded_files(
     settings, mocker, mocked_celery, mock_course_archive_bucket
 ):
-    """unpublish_all_staff_only_files fans out one task per chunk of course ids"""
-    mock_task = mocker.patch("learning_resources.tasks.unpublish_staff_only_files.si")
+    """unpublish_all_excluded_files fans out one task per chunk of course ids"""
+    mock_task = mocker.patch("learning_resources.tasks.unpublish_excluded_files.si")
     mocker.patch("learning_resources.tasks.load_course_blocklist", return_value=[])
     mocker.patch(
         "learning_resources.tasks.get_most_recent_course_archives",
@@ -1570,7 +1570,7 @@ def test_unpublish_all_staff_only_files(
         3, etl_source=etl_source, platform=PlatformType.mitxonline.name
     )
     with pytest.raises(mocked_celery.replace_exception_class):
-        tasks.unpublish_all_staff_only_files.delay(
+        tasks.unpublish_all_excluded_files.delay(
             etl_source=etl_source, chunk_size=2, learning_resource_ids=None
         )
     assert mock_task.call_count == 2
@@ -1578,13 +1578,18 @@ def test_unpublish_all_staff_only_files(
         rid for call in mock_task.call_args_list for rid in call.args[0]
     )
     assert called_ids == sorted(c.learning_resource_id for c in courses)
-    mock_task.assert_any_call(ANY, etl_source, ["foo.tar.gz"])
+    mock_task.assert_any_call(
+        ANY, etl_source, ["foo.tar.gz"], dry_run=False, report=False
+    )
 
 
-def test_unpublish_staff_only_files_task(mocker):
-    """unpublish_staff_only_files task delegates to edx_shared"""
+def test_unpublish_excluded_files_task(mocker):
+    """unpublish_excluded_files task delegates to edx_shared"""
     mock_fn = mocker.patch(
-        "learning_resources.tasks.unpublish_staff_only_content_files", return_value=3
+        "learning_resources.tasks.unpublish_excluded_content_files",
+        return_value=(3, []),
+    )
+    assert tasks.unpublish_excluded_files([1, 2], "mitxonline", ["k"]) == (3, [])
+    mock_fn.assert_called_once_with(
+        "mitxonline", [1, 2], ["k"], dry_run=False, report=False
     )
-    assert tasks.unpublish_staff_only_files([1, 2], "mitxonline", ["k"]) == 3
-    mock_fn.assert_called_once_with("mitxonline", [1, 2], ["k"])
diff --git a/main/celery.py b/main/celery.py
index 6a83648815..8fedd258f1 100644
--- a/main/celery.py
+++ b/main/celery.py
@@ -30,8 +30,8 @@
     "learning_resources.tasks.import_all_xpro_files": {"queue": "edx_content"},
     "learning_resources.tasks.import_all_mit_edx_files": {"queue": "edx_content"},
     "learning_resources.tasks.import_all_mitxonline_files": {"queue": "edx_content"},
-    "learning_resources.tasks.unpublish_staff_only_files": {"queue": "edx_content"},
-    "learning_resources.tasks.unpublish_all_staff_only_files": {"queue": "edx_content"},
+    "learning_resources.tasks.unpublish_excluded_files": {"queue": "edx_content"},
+    "learning_resources.tasks.unpublish_all_excluded_files": {"queue": "edx_content"},
     "learning_resources_search.tasks.index_run_content_files": {"queue": "edx_content"},
     "learning_resources_search.tasks.deindex_run_content_files": {
         "queue": "edx_content"

From d8e13f07c4c75abba1801febd823adad4129bf04 Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Tue, 15 Sep 2026 12:36:25 -0400
Subject: [PATCH 02/10] Report the full exclusion set in audit_olx_references

It only counted what the unreferenced-static filter drops, so it
under-reported what unpublish_excluded_files would actually unpublish -
it missed the static files caught by the staff-only set. It now walks
excluded_olx_paths, the same function ingestion and the cleanup use, and
splits the total by location and file class.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../commands/audit_olx_references.py          | 59 ++++++++++---------
 1 file changed, 31 insertions(+), 28 deletions(-)

diff --git a/learning_resources/management/commands/audit_olx_references.py b/learning_resources/management/commands/audit_olx_references.py
index c75da40f69..77c76537a5 100644
--- a/learning_resources/management/commands/audit_olx_references.py
+++ b/learning_resources/management/commands/audit_olx_references.py
@@ -1,12 +1,11 @@
 """Report what excluded_olx_paths would drop from an extracted OLX course tree"""
 
-from collections import Counter
 from pathlib import Path
 
 from django.core.management import BaseCommand
 
 from learning_resources.constants import VALID_TEXT_FILE_TYPES
-from learning_resources.etl.utils import staff_only_olx_paths, static_olx_references
+from learning_resources.etl.utils import excluded_olx_paths
 
 TRANSCRIPT_EXTENSIONS = (".srt", ".sjson", ".vtt")
 
@@ -18,9 +17,12 @@ class Command(BaseCommand):
 
         docker compose run --rm -v /path/to/extracted:/archives web \\
             ./manage.py audit_olx_references /archives/<course>
+
+    The excluded total is what unpublish_excluded_files would unpublish for a
+    run whose content files are fully ingested.
     """
 
-    help = "Report the static files an OLX course tree does not reference"
+    help = "Report the files an OLX course tree contains but does not use"
 
     def add_arguments(self, parser):
         parser.add_argument(
@@ -28,34 +30,35 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):  # noqa: ARG002
-        """Print per-archive drop counts split by file class"""
+        """Print per-archive exclusion counts split by location and file class"""
         for olx_path in options["olx_paths"]:
             root = Path(olx_path)
-            references = {
-                path: referrer
-                for path, referrer in static_olx_references(
-                    root, staff_only_olx_paths(root)
-                ).items()
-                if path.suffix.lower() in VALID_TEXT_FILE_TYPES
+            ingestable = {
+                path
+                for path in root.rglob("*")
+                if path.is_file()
+                and path.suffix.lower() in VALID_TEXT_FILE_TYPES
+                and not any(
+                    "draft" in part for part in path.relative_to(root).parts[:-1]
+                )
             }
-            dropped = [
-                path for path, referrer in references.items() if referrer is None
-            ]
-            transcripts = [
-                path for path in dropped if path.suffix.lower() in TRANSCRIPT_EXTENSIONS
-            ]
-            documents = [path for path in dropped if path not in set(transcripts)]
-            percent = 100 * len(dropped) / len(references) if references else 0
-            types = ", ".join(
-                f"{extension}:{count}"
-                for extension, count in Counter(
-                    path.suffix.lower() for path in documents
-                ).most_common(4)
-            )
+            excluded = excluded_olx_paths(root) & ingestable
+            static = {
+                path for path in excluded if path.relative_to(root).parts[0] == "static"
+            }
+            transcripts = {
+                path for path in static if path.suffix.lower() in TRANSCRIPT_EXTENSIONS
+            }
+            percent = 100 * len(excluded) / len(ingestable) if ingestable else 0
             self.stdout.write(f"\n=== {root}")
-            self.stdout.write(f"  ingestable static files : {len(references)}")
+            self.stdout.write(f"  ingestable files      : {len(ingestable)}")
+            self.stdout.write(
+                f"  excluded              : {len(excluded)} ({percent:.0f}%)"
+            )
+            self.stdout.write(f"    under static/       : {len(static)}")
+            self.stdout.write(f"      transcripts       : {len(transcripts)}")
+            self.stdout.write(f"      documents         : {len(static - transcripts)}")
             self.stdout.write(
-                f"  dropped                 : {len(dropped)} ({percent:.0f}%)"
+                f"    elsewhere           : {len(excluded - static)}"
+                "  (staff-only blocks, manifests, announcements)"
             )
-            self.stdout.write(f"    transcripts           : {len(transcripts)}")
-            self.stdout.write(f"    documents             : {len(documents)}  {types}")

From cfbd575b39c907c89cd6c565c523a7d46a634699 Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Tue, 15 Sep 2026 13:37:47 -0400
Subject: [PATCH 03/10] Drop the unpublish_excluded_files command test

The repo does not generally test management commands, and this one
collided with recreate_index_test on CI: learning_resources/management
has no __init__.py, so pytest named both modules commands.<name> and the
second import failed. edx_shared_test covers dry_run and the report rows
at the function level.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../commands/unpublish_excluded_files_test.py | 93 -------------------
 1 file changed, 93 deletions(-)
 delete mode 100644 learning_resources/management/commands/unpublish_excluded_files_test.py

diff --git a/learning_resources/management/commands/unpublish_excluded_files_test.py b/learning_resources/management/commands/unpublish_excluded_files_test.py
deleted file mode 100644
index 41b91ea29d..0000000000
--- a/learning_resources/management/commands/unpublish_excluded_files_test.py
+++ /dev/null
@@ -1,93 +0,0 @@
-"""Tests for the unpublish_excluded_files management command"""
-
-import csv
-from io import StringIO
-from pathlib import Path
-
-import pytest
-from django.core.management import CommandError, call_command
-
-pytestmark = pytest.mark.django_db
-
-COMMAND = "unpublish_excluded_files"
-TASK_PATH = (
-    "learning_resources.management.commands."
-    "unpublish_excluded_files.unpublish_all_excluded_files"
-)
-
-
-@pytest.fixture
-def mock_task(mocker):
-    """Mock the fan-out task, returning one chunk result per call"""
-    task = mocker.patch(TASK_PATH)
-    task.delay.return_value.get.return_value = [(2, []), (1, [])]
-    return task
-
-
-def test_unpublish_excluded_files_runs_every_source(mock_task):
-    """Without --source the command processes all four edX sources"""
-    stdout = StringIO()
-    call_command(COMMAND, stdout=stdout)
-
-    sources = [call.kwargs["etl_source"] for call in mock_task.delay.call_args_list]
-    assert sources == ["mitxonline", "mit_edx", "xpro", "oll"]
-    assert "mitxonline: unpublished 3 content files" in stdout.getvalue()
-
-
-def test_unpublish_excluded_files_dry_run(mock_task):
-    """--dry-run is passed to the task and reported as a would-be change"""
-    stdout = StringIO()
-    call_command(COMMAND, "--source", "xpro", "--dry-run", stdout=stdout)
-
-    assert mock_task.delay.call_args.kwargs["dry_run"] is True
-    assert "xpro: would unpublish 3 content files" in stdout.getvalue()
-
-
-def test_unpublish_excluded_files_report_requires_resource_ids(mock_task):
-    """A report would have to travel back through the result backend unbounded"""
-    with pytest.raises(CommandError, match="requires --resource-ids"):
-        call_command(COMMAND, "--report", "out.csv", stdout=StringIO())
-    mock_task.delay.assert_not_called()
-
-
-def test_unpublish_excluded_files_writes_report(mock_task, tmp_path):
-    """The command process writes the CSV, since the tasks run on other workers"""
-    mock_task.delay.return_value.get.return_value = [
-        (
-            1,
-            [
-                {
-                    "run_id": "course-v1:MITx+1+run",
-                    "key": "asset-v1:MITx+1+run+type@asset+block@old.pdf",
-                    "source_path": "static/old.pdf",
-                    "published": True,
-                }
-            ],
-        )
-    ]
-    report = tmp_path / "report.csv"
-    stdout = StringIO()
-    call_command(
-        COMMAND,
-        "--source",
-        "oll",
-        "--resource-ids",
-        "1,2",
-        "--report",
-        str(report),
-        stdout=stdout,
-    )
-
-    assert mock_task.delay.call_args.kwargs["report"] is True
-    with Path(report).open() as report_file:
-        rows = list(csv.DictReader(report_file))
-    assert rows == [
-        {
-            "etl_source": "oll",
-            "run_id": "course-v1:MITx+1+run",
-            "key": "asset-v1:MITx+1+run+type@asset+block@old.pdf",
-            "source_path": "static/old.pdf",
-            "published": "True",
-        }
-    ]
-    assert "Wrote 1 rows" in stdout.getvalue()

From 4cda9e163bcdbe6e619d9af0977d13701382551b Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Wed, 16 Sep 2026 08:51:46 -0400
Subject: [PATCH 04/10] Keep static files a visible block still links

A hidden video's transcripts go into the staff-only set, but u.lab keeps a
staff-only duplicate of many videos, so the live copy's transcripts went
with them: 279 transcripts of visible videos in 15.671.1x were excluded
even though static_olx_references named the live video block as their
referrer.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 learning_resources/etl/utils.py      | 11 ++++++----
 learning_resources/etl/utils_test.py | 33 ++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/learning_resources/etl/utils.py b/learning_resources/etl/utils.py
index a97f12be68..78501e731b 100644
--- a/learning_resources/etl/utils.py
+++ b/learning_resources/etl/utils.py
@@ -542,10 +542,13 @@ def excluded_olx_paths(olx_path: str | Path) -> set[Path]:
     excluded.update(
         root / name for name in NON_CONTENT_OLX_FILES if (root / name).is_file()
     )
-    excluded.update(
-        path
-        for path, referrer in static_olx_references(root, excluded).items()
-        if referrer is None
+    references = static_olx_references(root, excluded)
+    excluded.update(path for path, referrer in references.items() if referrer is None)
+    # A hidden video's transcripts are in the staff-only set, but the same file is
+    # often also the transcript of the visible copy of that video, so put back
+    # anything a visible block still links.
+    excluded.difference_update(
+        path for path, referrer in references.items() if referrer is not None
     )
     return excluded
 
diff --git a/learning_resources/etl/utils_test.py b/learning_resources/etl/utils_test.py
index 5ec25af549..3e635f24b8 100644
--- a/learning_resources/etl/utils_test.py
+++ b/learning_resources/etl/utils_test.py
@@ -387,6 +387,39 @@ def test_documents_from_olx_skips_staff_only_subtrees(tmp_path):
     ]
 
 
+def test_documents_from_olx_keeps_transcripts_a_visible_video_shares(tmp_path):
+    """A hidden copy of a video must not take the live copy's transcripts with it"""
+    olx = tmp_path / "course"
+    _write_olx(olx, "course.xml", '<course url_name="run" org="MITx" course="1"/>')
+    _write_olx(
+        olx,
+        "course/run.xml",
+        '<course><vertical url_name="v_ok"/>'
+        '<vertical url_name="v_staff" visible_to_staff_only="true"/></course>',
+    )
+    _write_olx(
+        olx, "vertical/v_ok.xml", '<vertical><video url_name="vid_ok"/></vertical>'
+    )
+    _write_olx(
+        olx, "vertical/v_staff.xml", '<vertical><video url_name="vid_old"/></vertical>'
+    )
+    shared = '<transcript language="en" src="shared.srt"/>'
+    _write_olx(olx, "video/vid_ok.xml", f'<video url_name="vid_ok">{shared}</video>')
+    _write_olx(
+        olx,
+        "video/vid_old.xml",
+        f'<video url_name="vid_old">{shared}'
+        '<transcript language="es" src="hidden_only.srt"/></video>',
+    )
+    _write_olx(olx, "static/shared.srt", "1\n00:00:00,000 --> 00:00:01,000\nshared\n")
+    _write_olx(olx, "static/hidden_only.srt", "1\n00:00:00,000 --> 00:00:01,000\nold\n")
+
+    excluded = utils.excluded_olx_paths(olx)
+    assert olx / "static/shared.srt" not in excluded
+    assert olx / "static/hidden_only.srt" in excluded
+    assert olx / "video/vid_old.xml" in excluded
+
+
 def test_documents_from_olx_skips_staff_only_inline_structure(tmp_path):
     """Structure held inline in course.xml (no pointer files) is still filtered"""
     olx = tmp_path / "course"

From 2abe849f40b1144f448c74e313451d5a0738efcd Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Wed, 16 Sep 2026 10:02:57 -0400
Subject: [PATCH 05/10] Hide a block only when every parent hides it

staff_only_olx_paths deduped blocks first-wins while walking, so a block
hanging under both a visible and a staff-only vertical was hidden or not
depending on stack order. In 15.671.1x that dropped video/ec404ea2 and the
six transcripts it declares, though a visible vertical holds it too.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 learning_resources/etl/utils.py      | 30 +++++++++++++++-------
 learning_resources/etl/utils_test.py | 37 ++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/learning_resources/etl/utils.py b/learning_resources/etl/utils.py
index 78501e731b..3a36c557b3 100644
--- a/learning_resources/etl/utils.py
+++ b/learning_resources/etl/utils.py
@@ -357,15 +357,12 @@ def staff_only_olx_paths(olx_path: str | Path) -> set[Path]:
     course = _parse_olx_block(root, "", "course")
     if course is None:
         return set()
-    hidden: set[Path] = set()
-    seen: set[tuple[str, str]] = set()
+    hidden: dict[tuple[str, str], set[Path]] = {}
+    visible: set[tuple[str, str]] = set()
+    seen: set[tuple[str, str, bool]] = set()
     stack = [(course, "course", course.get("url_name"), False)]
     while stack:
         pointer, tag, url_name, staff_only = stack.pop()
-        if url_name:
-            if (tag, url_name) in seen:
-                continue
-            seen.add((tag, url_name))
         # pointer file wins when present; otherwise the element is the block itself
         element = _parse_olx_block(root, tag, url_name) if url_name else None
         if element is None:
@@ -374,12 +371,27 @@ def staff_only_olx_paths(olx_path: str | Path) -> set[Path]:
             pointer.get("visible_to_staff_only"),
             element.get("visible_to_staff_only"),
         )
-        if staff_only and url_name:
-            hidden.update(_hidden_block_files(root, tag, url_name, element))
+        if url_name:
+            if (tag, url_name, staff_only) in seen:
+                continue
+            seen.add((tag, url_name, staff_only))
+            # a block can hang under two parents; seeing it anywhere a learner
+            # can reach makes it visible, whichever path the walk took first
+            if staff_only:
+                hidden[tag, url_name] = _hidden_block_files(
+                    root, tag, url_name, element
+                )
+            else:
+                visible.add((tag, url_name))
         stack.extend(
             (child, child.tag, child.get("url_name"), staff_only) for child in element
         )
-    return hidden
+    return {
+        path
+        for block, files in hidden.items()
+        if block not in visible
+        for path in files
+    }
 
 
 REFERENCE_SCAN_EXTENSIONS = frozenset({".xml", ".html", ".htm", ".json", ".txt", ".md"})
diff --git a/learning_resources/etl/utils_test.py b/learning_resources/etl/utils_test.py
index 3e635f24b8..a45fb91d23 100644
--- a/learning_resources/etl/utils_test.py
+++ b/learning_resources/etl/utils_test.py
@@ -420,6 +420,43 @@ def test_documents_from_olx_keeps_transcripts_a_visible_video_shares(tmp_path):
     assert olx / "video/vid_old.xml" in excluded
 
 
+def test_documents_from_olx_keeps_a_block_a_visible_parent_also_holds(tmp_path):
+    """A block under two parents is hidden only when every parent hides it"""
+    olx = tmp_path / "course"
+    _write_olx(olx, "course.xml", '<course url_name="run" org="MITx" course="1"/>')
+    _write_olx(
+        olx,
+        "course/run.xml",
+        '<course><vertical url_name="v_ok"/>'
+        '<vertical url_name="v_staff" visible_to_staff_only="true"/></course>',
+    )
+    shared = '<video url_name="vid_shared"/>'
+    _write_olx(olx, "vertical/v_ok.xml", f"<vertical>{shared}</vertical>")
+    _write_olx(
+        olx,
+        "vertical/v_staff.xml",
+        f'<vertical>{shared}<video url_name="vid_hidden"/></vertical>',
+    )
+    _write_olx(
+        olx,
+        "video/vid_shared.xml",
+        '<video url_name="vid_shared"><transcript language="en" src="shared.srt"/></video>',
+    )
+    _write_olx(
+        olx,
+        "video/vid_hidden.xml",
+        '<video url_name="vid_hidden"><transcript language="en" src="hidden.srt"/></video>',
+    )
+    _write_olx(olx, "static/shared.srt", "1\n00:00:00,000 --> 00:00:01,000\nshared\n")
+    _write_olx(olx, "static/hidden.srt", "1\n00:00:00,000 --> 00:00:01,000\nhidden\n")
+
+    excluded = utils.excluded_olx_paths(olx)
+    assert olx / "video/vid_shared.xml" not in excluded
+    assert olx / "static/shared.srt" not in excluded
+    assert olx / "video/vid_hidden.xml" in excluded
+    assert olx / "static/hidden.srt" in excluded
+
+
 def test_documents_from_olx_skips_staff_only_inline_structure(tmp_path):
     """Structure held inline in course.xml (no pointer files) is still filtered"""
     olx = tmp_path / "course"

From 6da5c116247d921e977d056461f2fe0670c79d34 Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Wed, 16 Sep 2026 10:31:50 -0400
Subject: [PATCH 06/10] Read every id in a multi-speed youtube attribute

The legacy attribute is a comma-separated "<speed>:<id>" list. Splitting
the whole value on ":" kept only the last entry's id, so transcripts named
for any other speed's id looked orphaned and were dropped.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 learning_resources/etl/utils.py                        | 10 ++++++----
 learning_resources/etl/utils_test.py                   |  1 +
 .../management/commands/audit_olx_references.py        |  5 +++--
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/learning_resources/etl/utils.py b/learning_resources/etl/utils.py
index 3a36c557b3..5a02f004c1 100644
--- a/learning_resources/etl/utils.py
+++ b/learning_resources/etl/utils.py
@@ -472,10 +472,12 @@ def _olx_video_ids(sources: list[Path]) -> set[str] | None:
         # iter() finds <video> whether it has its own file or sits inline
         for video in element.iter("video"):
             for attribute in VIDEO_ID_ATTRIBUTES:
-                # youtube is "<speed>:<id>", the others are bare ids
-                value = (video.get(attribute) or "").split(":")[-1]
-                if value:
-                    ids.add(normalize_asset_ref(value))
+                # youtube is a comma-separated "<speed>:<id>" list, the others
+                # are bare ids
+                for entry in (video.get(attribute) or "").split(","):
+                    value = entry.split(":")[-1].strip()
+                    if value:
+                        ids.add(normalize_asset_ref(value))
     return ids
 
 
diff --git a/learning_resources/etl/utils_test.py b/learning_resources/etl/utils_test.py
index a45fb91d23..404ed2c27c 100644
--- a/learning_resources/etl/utils_test.py
+++ b/learning_resources/etl/utils_test.py
@@ -632,6 +632,7 @@ def test_documents_from_olx_drafts_do_not_keep_assets(tmp_path):
         "<video url_name='vid' sub='AbC123'/>",
         '<video url_name="vid" sub = "AbC123"/>',
         '<video url_name="vid" youtube="1.00:AbC123"/>',
+        '<video url_name="vid" youtube="0.75:Fast,1.00:AbC123,1.50:Slow"/>',
         '<video url_name="vid" youtube_id_1_0="AbC123"/>',
     ],
 )
diff --git a/learning_resources/management/commands/audit_olx_references.py b/learning_resources/management/commands/audit_olx_references.py
index 77c76537a5..1e99b225eb 100644
--- a/learning_resources/management/commands/audit_olx_references.py
+++ b/learning_resources/management/commands/audit_olx_references.py
@@ -18,8 +18,9 @@ class Command(BaseCommand):
         docker compose run --rm -v /path/to/extracted:/archives web \\
             ./manage.py audit_olx_references /archives/<course>
 
-    The excluded total is what unpublish_excluded_files would unpublish for a
-    run whose content files are fully ingested.
+    The excluded total counts archive paths, so it is an upper bound on what
+    unpublish_excluded_files would unpublish rather than a row count: several
+    paths can collapse onto one ContentFile key.
     """
 
     help = "Report the files an OLX course tree contains but does not use"

From 440458b09294a8d2247758e40f5e28597d946586 Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Wed, 16 Sep 2026 10:53:07 -0400
Subject: [PATCH 07/10] Drop the CSV report path and the unread referrer names

audit_olx_references already answers what the CSV answered, off any
extracted tree with no DB, worker or --resource-ids, so the report flag
and the rows it threaded through the result backend can go. The return
type goes back to int.

static_olx_references returned the name of each file's referrer, but its
only caller tested it against None, so it now returns the referenced and
unreferenced sets instead.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 learning_resources/etl/edx_shared.py          | 31 ++------
 learning_resources/etl/edx_shared_test.py     | 70 ++++++-------------
 learning_resources/etl/utils.py               | 60 ++++++----------
 .../commands/unpublish_excluded_files.py      | 36 +---------
 learning_resources/tasks.py                   | 23 ++----
 learning_resources/tasks_test.py              | 12 ++--
 6 files changed, 60 insertions(+), 172 deletions(-)

diff --git a/learning_resources/etl/edx_shared.py b/learning_resources/etl/edx_shared.py
index f9bd0aa023..0b65db525b 100644
--- a/learning_resources/etl/edx_shared.py
+++ b/learning_resources/etl/edx_shared.py
@@ -370,13 +370,8 @@ def sync_edx_course_files(
 
 
 def unpublish_excluded_content_files(
-    etl_source: str,
-    ids: list[int],
-    keys: list[str],
-    *,
-    dry_run: bool = False,
-    report: bool = False,
-) -> tuple[int, list[dict]]:
+    etl_source: str, ids: list[int], keys: list[str], *, dry_run: bool = False
+) -> int:
     """
     Unpublish (and deindex) content files the course does not use — staff-only
     subtrees, asset manifests and unreferenced static files — for the runs
@@ -387,11 +382,9 @@ def unpublish_excluded_content_files(
         ids(list of int): list of course ids to process
         keys(list[str]): list of S3 archive keys to search through
         dry_run(bool): count the rows but leave them published and deindex nothing
-        report(bool): return a row per excluded file for the caller to write out
 
     Returns:
-        tuple of (int, list of dict): number of content files unpublished (or
-            that would be, under dry_run), and the report rows when requested
+        int: number of content files unpublished, or that would be under dry_run
     """
     from learning_resources_search import tasks as search_tasks
     from vector_search import tasks as vector_tasks
@@ -399,7 +392,6 @@ def unpublish_excluded_content_files(
     bucket = get_bucket_by_name(settings.COURSE_ARCHIVE_BUCKET_NAME)
     run_lookup = build_run_lookup(etl_source, ids)
     total = 0
-    rows = []
     for key in keys:
         matching_runs = run_lookup.get(extract_run_id_from_key(etl_source, key))
         if not matching_runs:
@@ -423,25 +415,12 @@ def unpublish_excluded_content_files(
                 log.exception("Malformed OLX in %s, skipping", key)
                 continue
             excluded_keys = {
-                get_edx_module_id(str(path), run): str(path.relative_to(olx_path))
-                for path in excluded_paths
+                get_edx_module_id(str(path), run) for path in excluded_paths
             }
         if not excluded_keys:
             continue
         # scoped to this run: keys embed the run_id, but never rely on that alone
         excluded_files = ContentFile.objects.filter(run=run, key__in=excluded_keys)
-        if report:
-            rows.extend(
-                {
-                    "run_id": run.run_id,
-                    "key": content_key,
-                    "source_path": excluded_keys.get(content_key, ""),
-                    "published": published,
-                }
-                for content_key, published in excluded_files.values_list(
-                    "key", "published"
-                )
-            )
         if dry_run:
             total += excluded_files.filter(published=True).count()
             continue
@@ -455,4 +434,4 @@ def unpublish_excluded_content_files(
         if excluded_files.exists():
             search_tasks.deindex_run_content_files.delay(run.id, unpublished_only=True)
             vector_tasks.remove_unpublished_run_content_files.delay(run.id)
-    return total, rows
+    return total
diff --git a/learning_resources/etl/edx_shared_test.py b/learning_resources/etl/edx_shared_test.py
index b5157d8ccd..6314fffe59 100644
--- a/learning_resources/etl/edx_shared_test.py
+++ b/learning_resources/etl/edx_shared_test.py
@@ -1730,7 +1730,7 @@ def test_unpublish_excluded_content_files(staff_only_run, mock_deindex_tasks):
     for cf_key in run_keys.values():
         ContentFileFactory.create(run=other_run, key=cf_key, published=True)
 
-    unpublished, _ = unpublish_excluded_content_files(
+    unpublished = unpublish_excluded_content_files(
         staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
     )
 
@@ -1755,9 +1755,12 @@ def test_unpublish_excluded_content_files_nothing_hidden(
         run=run, key=get_edx_module_id("course/html/h_ok.xml", run), published=True
     )
 
-    assert unpublish_excluded_content_files(
-        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
-    ) == (0, [])
+    assert (
+        unpublish_excluded_content_files(
+            staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
+        )
+        == 0
+    )
     mock_deindex_tasks.opensearch.assert_not_called()
 
 
@@ -1775,9 +1778,12 @@ def test_unpublish_excluded_content_files_malformed_archive(
         published=True,
     )
 
-    assert unpublish_excluded_content_files(
-        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
-    ) == (0, [])
+    assert (
+        unpublish_excluded_content_files(
+            staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
+        )
+        == 0
+    )
     assert ContentFile.objects.filter(run=staff_only_run.run, published=True).exists()
     mock_deindex_tasks.opensearch.assert_not_called()
 
@@ -1791,9 +1797,12 @@ def test_unpublish_excluded_content_files_rerun_redeindexes(
         run=run, key=get_edx_module_id("course/html/h_staff.xml", run), published=False
     )
 
-    assert unpublish_excluded_content_files(
-        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
-    ) == (0, [])
+    assert (
+        unpublish_excluded_content_files(
+            staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
+        )
+        == 0
+    )
     mock_deindex_tasks.opensearch.assert_called_once_with(run.id, unpublished_only=True)
     mock_deindex_tasks.qdrant.assert_called_once_with(run.id)
 
@@ -1806,7 +1815,7 @@ def test_unpublish_excluded_content_files_drops_unreferenced_static(
     stale_key = get_edx_module_id("course/static/stale_syllabus.pdf", run)
     ContentFileFactory.create(run=run, key=stale_key, published=True)
 
-    unpublished, _ = unpublish_excluded_content_files(
+    unpublished = unpublish_excluded_content_files(
         staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
     )
 
@@ -1825,7 +1834,7 @@ def test_unpublish_excluded_content_files_dry_run(staff_only_run, mock_deindex_t
         published=True,
     )
 
-    unpublished, _ = unpublish_excluded_content_files(
+    unpublished = unpublish_excluded_content_files(
         staff_only_run.source,
         [staff_only_run.course.id],
         [staff_only_run.key],
@@ -1836,40 +1845,3 @@ def test_unpublish_excluded_content_files_dry_run(staff_only_run, mock_deindex_t
     assert ContentFile.objects.filter(run=run, published=True).count() == 1
     mock_deindex_tasks.opensearch.assert_not_called()
     mock_deindex_tasks.qdrant.assert_not_called()
-
-
-def test_unpublish_excluded_content_files_report_rows(
-    staff_only_run, mock_deindex_tasks
-):
-    """Report rows name the run, key and source path of every excluded file"""
-    run = staff_only_run.run
-    stale_key = get_edx_module_id("course/static/stale_syllabus.pdf", run)
-    ContentFileFactory.create(run=run, key=stale_key, published=True)
-
-    _, rows = unpublish_excluded_content_files(
-        staff_only_run.source,
-        [staff_only_run.course.id],
-        [staff_only_run.key],
-        dry_run=True,
-        report=True,
-    )
-
-    assert [row["key"] for row in rows] == [stale_key]
-    assert rows[0]["run_id"] == run.run_id
-    assert rows[0]["source_path"] == "static/stale_syllabus.pdf"
-    assert rows[0]["published"] is True
-
-
-def test_unpublish_excluded_content_files_no_report_by_default(
-    staff_only_run, mock_deindex_tasks
-):
-    """Rows are only collected when asked for, since they cross the result backend"""
-    ContentFileFactory.create(
-        run=staff_only_run.run,
-        key=get_edx_module_id("course/static/stale_syllabus.pdf", staff_only_run.run),
-        published=True,
-    )
-    _, rows = unpublish_excluded_content_files(
-        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
-    )
-    assert rows == []
diff --git a/learning_resources/etl/utils.py b/learning_resources/etl/utils.py
index 5a02f004c1..4770a5482e 100644
--- a/learning_resources/etl/utils.py
+++ b/learning_resources/etl/utils.py
@@ -404,7 +404,6 @@ def staff_only_olx_paths(olx_path: str | Path) -> set[Path]:
     "assets/assets.xml",
     "info/updates.items.json",
 )
-COURSE_UPDATES_FILE = "info/updates.items.json"
 
 # A legacy transcript is named for its video's id rather than for anything the
 # course text contains, so the id is the only link back to the block using it.
@@ -444,7 +443,9 @@ def _live_course_updates(root: Path) -> str:
     the export, but a live one still counts.
     """
     try:
-        items = json.loads((root / COURSE_UPDATES_FILE).read_text(errors="ignore"))
+        items = json.loads(
+            (root / "info/updates.items.json").read_text(errors="ignore")
+        )
     except (OSError, ValueError):
         return ""
     return "\n".join(
@@ -481,10 +482,10 @@ def _olx_video_ids(sources: list[Path]) -> set[str] | None:
     return ids
 
 
-def static_olx_references(root: Path, skip: set[Path]) -> dict[Path, str | None]:
+def static_olx_references(root: Path, skip: set[Path]) -> tuple[set[Path], set[Path]]:
     """
-    Map every file under static/ to the source file that refers to it, or None
-    when nothing in the course does.
+    Split the files under static/ into the ones the course refers to and the
+    ones nothing in it does.
 
     Matching is on the filename rather than on a "/static/" prefix, because
     courses also link assets as "asset-v1:...+type@asset+block/<name>", and it
@@ -497,43 +498,30 @@ def static_olx_references(root: Path, skip: set[Path]) -> dict[Path, str | None]
             staff-only set, so an asset only an answer key mentions is unreferenced
 
     Returns:
-        dict of Path to str or None: static file -> relative path of its referrer
+        tuple of (set of Path, set of Path): referenced and unreferenced static files
     """
     static_dir = root / "static"
     if not static_dir.is_dir():
-        return {}
+        return set(), set()
     sources = _olx_reference_sources(root, skip)
-    texts = [
-        (
-            path.relative_to(root).as_posix(),
-            normalize_asset_ref(path.read_text(errors="ignore")),
-        )
-        for path in sources
-    ]
-    updates = normalize_asset_ref(_live_course_updates(root))
-    if updates:
-        texts.append((COURSE_UPDATES_FILE, updates))
-    blob = "\n".join(text for _, text in texts)
+    blob = "\n".join(
+        [normalize_asset_ref(path.read_text(errors="ignore")) for path in sources]
+        + [normalize_asset_ref(_live_course_updates(root))]
+    )
     video_ids = _olx_video_ids(sources)
 
-    references = {}
+    referenced, unreferenced = set(), set()
     for path in sorted(static_dir.rglob("*")):
         if not path.is_file():
             continue
-        name = normalize_asset_ref(path.name)
-        if name in blob:
-            references[path] = next(
-                (source for source, text in texts if name in text), None
-            )
-            continue
-        legacy = LEGACY_TRANSCRIPT_RE.match(path.name)
-        if legacy and (
-            video_ids is None or normalize_asset_ref(legacy.group(1)) in video_ids
+        if normalize_asset_ref(path.name) in blob or (
+            (legacy := LEGACY_TRANSCRIPT_RE.match(path.name))
+            and (video_ids is None or normalize_asset_ref(legacy.group(1)) in video_ids)
         ):
-            references[path] = "video block id"
-            continue
-        references[path] = None
-    return references
+            referenced.add(path)
+        else:
+            unreferenced.add(path)
+    return referenced, unreferenced
 
 
 def excluded_olx_paths(olx_path: str | Path) -> set[Path]:
@@ -556,14 +544,12 @@ def excluded_olx_paths(olx_path: str | Path) -> set[Path]:
     excluded.update(
         root / name for name in NON_CONTENT_OLX_FILES if (root / name).is_file()
     )
-    references = static_olx_references(root, excluded)
-    excluded.update(path for path, referrer in references.items() if referrer is None)
+    referenced, unreferenced = static_olx_references(root, excluded)
+    excluded.update(unreferenced)
     # A hidden video's transcripts are in the staff-only set, but the same file is
     # often also the transcript of the visible copy of that video, so put back
     # anything a visible block still links.
-    excluded.difference_update(
-        path for path, referrer in references.items() if referrer is not None
-    )
+    excluded.difference_update(referenced)
     return excluded
 
 
diff --git a/learning_resources/management/commands/unpublish_excluded_files.py b/learning_resources/management/commands/unpublish_excluded_files.py
index 9a19411d0e..02bad11eb6 100644
--- a/learning_resources/management/commands/unpublish_excluded_files.py
+++ b/learning_resources/management/commands/unpublish_excluded_files.py
@@ -1,9 +1,6 @@
 """Unpublish edX content files that the course itself does not use"""
 
-import csv
-from pathlib import Path
-
-from django.core.management import BaseCommand, CommandError
+from django.core.management import BaseCommand
 
 from learning_resources.etl.constants import ETLSource
 from learning_resources.tasks import unpublish_all_excluded_files
@@ -16,7 +13,6 @@
     ETLSource.xpro.name,
     ETLSource.oll.name,
 ]
-REPORT_FIELDS = ("etl_source", "run_id", "key", "source_path", "published")
 
 
 class Command(BaseCommand):
@@ -56,16 +52,6 @@ def add_arguments(self, parser):
             action="store_true",
             help="Report what would be unpublished without changing anything",
         )
-        parser.add_argument(
-            "--report",
-            dest="report",
-            required=False,
-            help=(
-                "Write a CSV of every excluded content file to this path. "
-                "Requires --resource-ids, since the rows travel back through "
-                "the celery result backend"
-            ),
-        )
 
     def handle(self, *args, **options):  # noqa: ARG002
         """Run the unpublish tasks"""
@@ -74,35 +60,17 @@ def handle(self, *args, **options):  # noqa: ARG002
             if options["learning_resource_ids"]
             else None
         )
-        report_path = options["report"]
-        if report_path and not resource_ids:
-            msg = "--report requires --resource-ids"
-            raise CommandError(msg)
-
         start = now_in_utc()
-        rows = []
         for source in options["sources"] or EDX_SOURCES:
             task = unpublish_all_excluded_files.delay(
                 etl_source=source,
                 chunk_size=options["chunk_size"],
                 learning_resource_ids=resource_ids,
                 dry_run=options["dry_run"],
-                report=bool(report_path),
             )
             self.stdout.write(f"Started task {task} for {source}, waiting...")
-            total = 0
-            for count, source_rows in task.get() or []:
-                total += count or 0
-                rows.extend({"etl_source": source, **row} for row in source_rows or [])
+            total = sum(count or 0 for count in task.get() or [])
             verb = "would unpublish" if options["dry_run"] else "unpublished"
             self.stdout.write(f"{source}: {verb} {total} content files")
-
-        if report_path:
-            with Path(report_path).open("w", newline="") as report_file:
-                writer = csv.DictWriter(report_file, fieldnames=REPORT_FIELDS)
-                writer.writeheader()
-                writer.writerows(rows)
-            self.stdout.write(f"Wrote {len(rows)} rows to {report_path}")
-
         total_seconds = (now_in_utc() - start).total_seconds()
         self.stdout.write(f"Finished in {total_seconds} seconds")
diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py
index cdb7711caf..f137cfc310 100644
--- a/learning_resources/tasks.py
+++ b/learning_resources/tasks.py
@@ -233,28 +233,15 @@ def _content_file_resource_ids(etl_source: str, learning_resource_ids):
 
 @app.task(acks_late=True, reject_on_worker_lost=True)
 def unpublish_excluded_files(
-    ids: list[int],
-    etl_source: str,
-    keys: list[str],
-    *,
-    dry_run: bool = False,
-    report: bool = False,
+    ids: list[int], etl_source: str, keys: list[str], *, dry_run: bool = False
 ):
     """Unpublish unused content files for a chunk of courses"""
-    return unpublish_excluded_content_files(
-        etl_source, ids, keys, dry_run=dry_run, report=report
-    )
+    return unpublish_excluded_content_files(etl_source, ids, keys, dry_run=dry_run)
 
 
 @app.task(bind=True)
-def unpublish_all_excluded_files(  # noqa: PLR0913
-    self,
-    *,
-    etl_source,
-    chunk_size=None,
-    learning_resource_ids=None,
-    dry_run=False,
-    report=False,
+def unpublish_all_excluded_files(
+    self, *, etl_source, chunk_size=None, learning_resource_ids=None, dry_run=False
 ):
     """Fan out unpublish_excluded_files over an edX source's current archives"""
     if chunk_size is None:
@@ -264,7 +251,7 @@ def unpublish_all_excluded_files(  # noqa: PLR0913
         celery.group(
             [
                 unpublish_excluded_files.si(
-                    ids, etl_source, archive_keys, dry_run=dry_run, report=report
+                    ids, etl_source, archive_keys, dry_run=dry_run
                 )
                 for ids in chunks(
                     _content_file_resource_ids(etl_source, learning_resource_ids),
diff --git a/learning_resources/tasks_test.py b/learning_resources/tasks_test.py
index 2fded2555a..ae1e4e0a6b 100644
--- a/learning_resources/tasks_test.py
+++ b/learning_resources/tasks_test.py
@@ -1578,18 +1578,14 @@ def test_unpublish_all_excluded_files(
         rid for call in mock_task.call_args_list for rid in call.args[0]
     )
     assert called_ids == sorted(c.learning_resource_id for c in courses)
-    mock_task.assert_any_call(
-        ANY, etl_source, ["foo.tar.gz"], dry_run=False, report=False
-    )
+    mock_task.assert_any_call(ANY, etl_source, ["foo.tar.gz"], dry_run=False)
 
 
 def test_unpublish_excluded_files_task(mocker):
     """unpublish_excluded_files task delegates to edx_shared"""
     mock_fn = mocker.patch(
         "learning_resources.tasks.unpublish_excluded_content_files",
-        return_value=(3, []),
-    )
-    assert tasks.unpublish_excluded_files([1, 2], "mitxonline", ["k"]) == (3, [])
-    mock_fn.assert_called_once_with(
-        "mitxonline", [1, 2], ["k"], dry_run=False, report=False
+        return_value=3,
     )
+    assert tasks.unpublish_excluded_files([1, 2], "mitxonline", ["k"]) == 3
+    mock_fn.assert_called_once_with("mitxonline", [1, 2], ["k"], dry_run=False)

From 78e1e2b6bb94fe9488068ecc9928e70a67a3b6fd Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Wed, 16 Sep 2026 11:13:45 -0400
Subject: [PATCH 08/10] Report per-run exclusion counts, and optionally as CSV
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

unpublish_excluded_content_files now returns a row per run — run_id,
excluded, unpublished and the run's total content files — instead of a
bare count, so the command can print a line per run and a summary per
source. Counts rather than paths, so the payload is bounded by run count
and crosses the result backend at any scale, which is what the old
per-file CSV could not do.

--report writes the same rows as CSV, and no longer needs --resource-ids.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 learning_resources/etl/edx_shared.py          | 42 ++++++++++++-------
 learning_resources/etl/edx_shared_test.py     | 26 ++++++------
 .../commands/unpublish_excluded_files.py      | 40 +++++++++++++++++-
 learning_resources/tasks.py                   |  2 +-
 learning_resources/tasks_test.py              |  6 ++-
 5 files changed, 84 insertions(+), 32 deletions(-)

diff --git a/learning_resources/etl/edx_shared.py b/learning_resources/etl/edx_shared.py
index 0b65db525b..0b77dc265e 100644
--- a/learning_resources/etl/edx_shared.py
+++ b/learning_resources/etl/edx_shared.py
@@ -371,7 +371,7 @@ def sync_edx_course_files(
 
 def unpublish_excluded_content_files(
     etl_source: str, ids: list[int], keys: list[str], *, dry_run: bool = False
-) -> int:
+) -> list[dict]:
     """
     Unpublish (and deindex) content files the course does not use — staff-only
     subtrees, asset manifests and unreferenced static files — for the runs
@@ -384,14 +384,18 @@ def unpublish_excluded_content_files(
         dry_run(bool): count the rows but leave them published and deindex nothing
 
     Returns:
-        int: number of content files unpublished, or that would be under dry_run
+        list of dict: a row per run whose archive excludes content files it has,
+            counting the excluded rows, the ones this call unpublished (or would
+            have, under dry_run) and the run's content files in total. Counts,
+            not paths, so the payload stays small enough to cross the celery
+            result backend for every run at once.
     """
     from learning_resources_search import tasks as search_tasks
     from vector_search import tasks as vector_tasks
 
     bucket = get_bucket_by_name(settings.COURSE_ARCHIVE_BUCKET_NAME)
     run_lookup = build_run_lookup(etl_source, ids)
-    total = 0
+    rows = []
     for key in keys:
         matching_runs = run_lookup.get(extract_run_id_from_key(etl_source, key))
         if not matching_runs:
@@ -421,17 +425,27 @@ def unpublish_excluded_content_files(
             continue
         # scoped to this run: keys embed the run_id, but never rely on that alone
         excluded_files = ContentFile.objects.filter(run=run, key__in=excluded_keys)
-        if dry_run:
-            total += excluded_files.filter(published=True).count()
+        excluded = excluded_files.count()
+        if not excluded:
             continue
-        unpublished = excluded_files.filter(published=True).update(published=False)
-        total += unpublished
-        log.info(
-            "Unpublished %d excluded content files for %s", unpublished, run.run_id
-        )
-        # dispatched whenever excluded rows exist, not only when this call flipped
-        # them, so a re-run after a failed deindex task cleans up the indexes
-        if excluded_files.exists():
+        if dry_run:
+            unpublished = excluded_files.filter(published=True).count()
+        else:
+            unpublished = excluded_files.filter(published=True).update(published=False)
+            log.info(
+                "Unpublished %d excluded content files for %s", unpublished, run.run_id
+            )
+            # dispatched whenever excluded rows exist, not only when this call
+            # flipped them, so a re-run after a failed deindex task cleans up
+            # the indexes
             search_tasks.deindex_run_content_files.delay(run.id, unpublished_only=True)
             vector_tasks.remove_unpublished_run_content_files.delay(run.id)
-    return total
+        rows.append(
+            {
+                "run_id": run.run_id,
+                "excluded": excluded,
+                "unpublished": unpublished,
+                "total": ContentFile.objects.filter(run=run).count(),
+            }
+        )
+    return rows
diff --git a/learning_resources/etl/edx_shared_test.py b/learning_resources/etl/edx_shared_test.py
index 6314fffe59..51d58c374f 100644
--- a/learning_resources/etl/edx_shared_test.py
+++ b/learning_resources/etl/edx_shared_test.py
@@ -1730,11 +1730,11 @@ def test_unpublish_excluded_content_files(staff_only_run, mock_deindex_tasks):
     for cf_key in run_keys.values():
         ContentFileFactory.create(run=other_run, key=cf_key, published=True)
 
-    unpublished = unpublish_excluded_content_files(
+    rows = unpublish_excluded_content_files(
         staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
     )
 
-    assert unpublished == 1
+    assert [row["unpublished"] for row in rows] == [1]
     assert not ContentFile.objects.filter(
         run=run, key=run_keys["html/h_staff.xml"], published=True
     ).exists()
@@ -1759,7 +1759,7 @@ def test_unpublish_excluded_content_files_nothing_hidden(
         unpublish_excluded_content_files(
             staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
         )
-        == 0
+        == []
     )
     mock_deindex_tasks.opensearch.assert_not_called()
 
@@ -1782,7 +1782,7 @@ def test_unpublish_excluded_content_files_malformed_archive(
         unpublish_excluded_content_files(
             staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
         )
-        == 0
+        == []
     )
     assert ContentFile.objects.filter(run=staff_only_run.run, published=True).exists()
     mock_deindex_tasks.opensearch.assert_not_called()
@@ -1797,12 +1797,12 @@ def test_unpublish_excluded_content_files_rerun_redeindexes(
         run=run, key=get_edx_module_id("course/html/h_staff.xml", run), published=False
     )
 
-    assert (
-        unpublish_excluded_content_files(
-            staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
-        )
-        == 0
+    rows = unpublish_excluded_content_files(
+        staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
     )
+
+    # still counted as excluded, but this call had nothing left to flip
+    assert rows == [{"run_id": run.run_id, "excluded": 1, "unpublished": 0, "total": 1}]
     mock_deindex_tasks.opensearch.assert_called_once_with(run.id, unpublished_only=True)
     mock_deindex_tasks.qdrant.assert_called_once_with(run.id)
 
@@ -1815,11 +1815,11 @@ def test_unpublish_excluded_content_files_drops_unreferenced_static(
     stale_key = get_edx_module_id("course/static/stale_syllabus.pdf", run)
     ContentFileFactory.create(run=run, key=stale_key, published=True)
 
-    unpublished = unpublish_excluded_content_files(
+    rows = unpublish_excluded_content_files(
         staff_only_run.source, [staff_only_run.course.id], [staff_only_run.key]
     )
 
-    assert unpublished == 1
+    assert [row["unpublished"] for row in rows] == [1]
     assert not ContentFile.objects.filter(
         run=run, key=stale_key, published=True
     ).exists()
@@ -1834,14 +1834,14 @@ def test_unpublish_excluded_content_files_dry_run(staff_only_run, mock_deindex_t
         published=True,
     )
 
-    unpublished = unpublish_excluded_content_files(
+    rows = unpublish_excluded_content_files(
         staff_only_run.source,
         [staff_only_run.course.id],
         [staff_only_run.key],
         dry_run=True,
     )
 
-    assert unpublished == 1
+    assert [row["unpublished"] for row in rows] == [1]
     assert ContentFile.objects.filter(run=run, published=True).count() == 1
     mock_deindex_tasks.opensearch.assert_not_called()
     mock_deindex_tasks.qdrant.assert_not_called()
diff --git a/learning_resources/management/commands/unpublish_excluded_files.py b/learning_resources/management/commands/unpublish_excluded_files.py
index 02bad11eb6..ffc6d4147a 100644
--- a/learning_resources/management/commands/unpublish_excluded_files.py
+++ b/learning_resources/management/commands/unpublish_excluded_files.py
@@ -1,5 +1,9 @@
 """Unpublish edX content files that the course itself does not use"""
 
+import csv
+from operator import itemgetter
+from pathlib import Path
+
 from django.core.management import BaseCommand
 
 from learning_resources.etl.constants import ETLSource
@@ -15,6 +19,13 @@
 ]
 
 
+REPORT_FIELDS = ("etl_source", "run_id", "excluded", "unpublished", "total")
+
+
+def _sum(rows, field):
+    return sum(row[field] for row in rows)
+
+
 class Command(BaseCommand):
     """
     Walk each course's current archive and unpublish the content files it
@@ -52,6 +63,12 @@ def add_arguments(self, parser):
             action="store_true",
             help="Report what would be unpublished without changing anything",
         )
+        parser.add_argument(
+            "--report",
+            dest="report",
+            required=False,
+            help="Also write the per-run counts as CSV to this path",
+        )
 
     def handle(self, *args, **options):  # noqa: ARG002
         """Run the unpublish tasks"""
@@ -61,6 +78,7 @@ def handle(self, *args, **options):  # noqa: ARG002
             else None
         )
         start = now_in_utc()
+        report = []
         for source in options["sources"] or EDX_SOURCES:
             task = unpublish_all_excluded_files.delay(
                 etl_source=source,
@@ -69,8 +87,26 @@ def handle(self, *args, **options):  # noqa: ARG002
                 dry_run=options["dry_run"],
             )
             self.stdout.write(f"Started task {task} for {source}, waiting...")
-            total = sum(count or 0 for count in task.get() or [])
+            rows = [row for chunk in task.get() or [] for row in chunk or []]
+            for row in sorted(rows, key=itemgetter("run_id")):
+                self.stdout.write(
+                    f"{source} run {row['run_id']}: {row['excluded']} out of "
+                    f"{row['total']} content files excluded"
+                )
             verb = "would unpublish" if options["dry_run"] else "unpublished"
-            self.stdout.write(f"{source}: {verb} {total} content files")
+            self.stdout.write(
+                f"{source} summary: {_sum(rows, 'excluded')} out of "
+                f"{_sum(rows, 'total')} content files excluded across "
+                f"{len(rows)} runs, {verb} {_sum(rows, 'unpublished')}"
+            )
+            report.extend({"etl_source": source, **row} for row in rows)
+
+        if options["report"]:
+            with Path(options["report"]).open("w", newline="") as report_file:
+                writer = csv.DictWriter(report_file, fieldnames=REPORT_FIELDS)
+                writer.writeheader()
+                writer.writerows(sorted(report, key=itemgetter("etl_source", "run_id")))
+            self.stdout.write(f"Wrote {len(report)} rows to {options['report']}")
+
         total_seconds = (now_in_utc() - start).total_seconds()
         self.stdout.write(f"Finished in {total_seconds} seconds")
diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py
index f137cfc310..639de0fbfc 100644
--- a/learning_resources/tasks.py
+++ b/learning_resources/tasks.py
@@ -235,7 +235,7 @@ def _content_file_resource_ids(etl_source: str, learning_resource_ids):
 def unpublish_excluded_files(
     ids: list[int], etl_source: str, keys: list[str], *, dry_run: bool = False
 ):
-    """Unpublish unused content files for a chunk of courses"""
+    """Unpublish unused content files for a chunk of courses, a row per run"""
     return unpublish_excluded_content_files(etl_source, ids, keys, dry_run=dry_run)
 
 
diff --git a/learning_resources/tasks_test.py b/learning_resources/tasks_test.py
index ae1e4e0a6b..7ef9f38215 100644
--- a/learning_resources/tasks_test.py
+++ b/learning_resources/tasks_test.py
@@ -1585,7 +1585,9 @@ def test_unpublish_excluded_files_task(mocker):
     """unpublish_excluded_files task delegates to edx_shared"""
     mock_fn = mocker.patch(
         "learning_resources.tasks.unpublish_excluded_content_files",
-        return_value=3,
+        return_value=[{"run_id": "r", "excluded": 3, "unpublished": 3, "total": 9}],
     )
-    assert tasks.unpublish_excluded_files([1, 2], "mitxonline", ["k"]) == 3
+    assert tasks.unpublish_excluded_files([1, 2], "mitxonline", ["k"]) == [
+        {"run_id": "r", "excluded": 3, "unpublished": 3, "total": 9}
+    ]
     mock_fn.assert_called_once_with("mitxonline", [1, 2], ["k"], dry_run=False)

From 74c40ee4db73e23a3ecf58342b2c7acef28c41dd Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Wed, 16 Sep 2026 11:24:03 -0400
Subject: [PATCH 09/10] Pluralize the run count in the summary line

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 .../management/commands/unpublish_excluded_files.py             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/learning_resources/management/commands/unpublish_excluded_files.py b/learning_resources/management/commands/unpublish_excluded_files.py
index ffc6d4147a..f261380a44 100644
--- a/learning_resources/management/commands/unpublish_excluded_files.py
+++ b/learning_resources/management/commands/unpublish_excluded_files.py
@@ -97,7 +97,7 @@ def handle(self, *args, **options):  # noqa: ARG002
             self.stdout.write(
                 f"{source} summary: {_sum(rows, 'excluded')} out of "
                 f"{_sum(rows, 'total')} content files excluded across "
-                f"{len(rows)} runs, {verb} {_sum(rows, 'unpublished')}"
+                f"{len(rows)} run(s), {verb} {_sum(rows, 'unpublished')}"
             )
             report.extend({"etl_source": source, **row} for row in rows)
 

From b795df6e5085cb7e326a48e999099211d9bba249 Mon Sep 17 00:00:00 2001
From: Matt Bertrand <mattbert@mit.edu>
Date: Wed, 16 Sep 2026 13:59:37 -0400
Subject: [PATCH 10/10] Skip runs with no content files when unpublishing

Unscoped, the fan-out selected every published course for the source and
downloaded each one's archive just to find the run has nothing to
unpublish. A run with no content files can't have any excluded, so filter
those out before chunking. Deliberately not pushed into
_content_file_resource_ids, which the ingestion fan-out shares: there a
course with no content files yet is the one whose archive must be read.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
---
 learning_resources/tasks.py      | 14 ++++++++++----
 learning_resources/tasks_test.py | 14 +++++++++++---
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py
index 639de0fbfc..73a0224ed0 100644
--- a/learning_resources/tasks.py
+++ b/learning_resources/tasks.py
@@ -247,16 +247,22 @@ def unpublish_all_excluded_files(
     if chunk_size is None:
         chunk_size = settings.LEARNING_COURSE_ITERATOR_CHUNK_SIZE
     archive_keys = get_most_recent_course_archives(etl_source)
+    # a run with no content files has none to unpublish, and skipping it saves
+    # downloading and extracting its archive to find that out. Not applied to
+    # the ingestion fan-out, where a course with no content files yet is
+    # exactly the one that needs its archive read.
+    resource_ids = (
+        _content_file_resource_ids(etl_source, learning_resource_ids)
+        .filter(runs__content_files__isnull=False)
+        .distinct()
+    )
     return self.replace(
         celery.group(
             [
                 unpublish_excluded_files.si(
                     ids, etl_source, archive_keys, dry_run=dry_run
                 )
-                for ids in chunks(
-                    _content_file_resource_ids(etl_source, learning_resource_ids),
-                    chunk_size=chunk_size,
-                )
+                for ids in chunks(resource_ids, chunk_size=chunk_size)
             ]
         )
     )
diff --git a/learning_resources/tasks_test.py b/learning_resources/tasks_test.py
index 7ef9f38215..47b24e0bb3 100644
--- a/learning_resources/tasks_test.py
+++ b/learning_resources/tasks_test.py
@@ -1557,7 +1557,7 @@ def test_get_podcast_transcripts(mocker):
 def test_unpublish_all_excluded_files(
     settings, mocker, mocked_celery, mock_course_archive_bucket
 ):
-    """unpublish_all_excluded_files fans out one task per chunk of course ids"""
+    """Only courses whose runs have content files are fanned out"""
     mock_task = mocker.patch("learning_resources.tasks.unpublish_excluded_files.si")
     mocker.patch("learning_resources.tasks.load_course_blocklist", return_value=[])
     mocker.patch(
@@ -1569,15 +1569,23 @@ def test_unpublish_all_excluded_files(
     courses = factories.CourseFactory.create_batch(
         3, etl_source=etl_source, platform=PlatformType.mitxonline.name
     )
+    # the third course has no content files, so its archive is never downloaded
+    with_files = courses[:2]
+    for course in with_files:
+        factories.ContentFileFactory.create(
+            run=factories.LearningResourceRunFactory.create(
+                learning_resource=course.learning_resource
+            )
+        )
     with pytest.raises(mocked_celery.replace_exception_class):
         tasks.unpublish_all_excluded_files.delay(
             etl_source=etl_source, chunk_size=2, learning_resource_ids=None
         )
-    assert mock_task.call_count == 2
+    assert mock_task.call_count == 1
     called_ids = sorted(
         rid for call in mock_task.call_args_list for rid in call.args[0]
     )
-    assert called_ids == sorted(c.learning_resource_id for c in courses)
+    assert called_ids == sorted(c.learning_resource_id for c in with_files)
     mock_task.assert_any_call(ANY, etl_source, ["foo.tar.gz"], dry_run=False)