Skip to content

Commit 41f91d5

Browse files
nodeeeeeeclaude
andcommitted
Fix artifact leaks, multi-part overwrite, and add threaded pipeline
Artifact cleanup: - _clean_artifacts() strips "APPROVED", "NUS Confidential", and "© CS2105" from generated drafts before caching - _BAD_LABEL regex rejects these as section titles - Fallback title is now "Slides N–M" instead of first (bad) label Multi-part lecture fix: - Per-video mode now groups lectures by number and merges parts into one file. Previously, L06 Part 2 overwrote L06 Part 1 since both wrote to CS2105_L06_notes.md. Now both parts are combined. - Source metadata lists all parts with their video/slide sources. Threaded pipeline: - New pipeline_worker.py with producer-consumer threading - Pipeline page uses it instead of 3 sequential steps Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 27c4610 commit 41f91d5

2 files changed

Lines changed: 89 additions & 49 deletions

File tree

note_generation.py

Lines changed: 87 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -648,8 +648,29 @@ def render_slide_images(slide_path: Path, out_dir: Path,
648648

649649
# ── Chunk helpers ─────────────────────────────────────────────────────────────
650650

651+
def _clean_artifacts(text: str) -> str:
652+
"""Remove pipeline artifacts that may leak into generated notes."""
653+
lines = text.splitlines()
654+
cleaned = []
655+
for line in lines:
656+
stripped = line.strip()
657+
# Remove bare "APPROVED" lines (verifier leak)
658+
if stripped == "APPROVED":
659+
continue
660+
# Remove verify prompt leaks
661+
if "reply APPROVED (this word only)" in stripped:
662+
continue
663+
# Clean section-header artifacts
664+
line = re.sub(r"##\s*NUS Confidential\s*##", "", line)
665+
line = re.sub(r"[©(]\s*c?\)?\s*CS\d+", "", line)
666+
cleaned.append(line)
667+
return "\n".join(cleaned)
668+
669+
651670
_BAD_LABEL = re.compile(
652-
r"^\s*(\d+|[A-Z]{2,4}\d{4}[\s\-].*|CS\d+.*|AY\d+.*|\[.*\])\s*$"
671+
r"^\s*(\d+|[A-Z]{2,4}\d{4}[\s\-].*|CS\d+.*|AY\d+.*|\[.*\]|"
672+
r".*NUS Confidential.*|.*©\s*CS\d+.*|\(c\)\s*CS\d+.*|Page\s+\d+)\s*$",
673+
re.IGNORECASE,
653674
)
654675

655676
def _chunk_title(slides_in_chunk: list[SlideInfo]) -> str:
@@ -677,7 +698,10 @@ def _is_good(label: str) -> bool:
677698
for s in slides_in_chunk:
678699
if _is_good(s.label):
679700
return s.label
680-
return slides_in_chunk[0].label
701+
# Last resort: use slide range as title instead of a bad label
702+
first = slides_in_chunk[0].index + 1
703+
last = slides_in_chunk[-1].index + 1
704+
return f"Slides {first}{last}" if first != last else f"Slide {first}"
681705

682706

683707
def _build_chunk_prompt(
@@ -846,6 +870,9 @@ def generate_section(
846870
else:
847871
tqdm.write(f" [warn] Verifier suspicious response, keeping draft")
848872

873+
# Strip pipeline artifacts that may have leaked into the draft
874+
draft = _clean_artifacts(draft)
875+
849876
# Translate to target language if not English
850877
if NOTE_LANGUAGE != "en" and draft:
851878
lang = _LANG_NAMES.get(NOTE_LANGUAGE, NOTE_LANGUAGE)
@@ -1270,11 +1297,14 @@ def generate_per_video_notes(
12701297
fmt: str = OUTPUT_FORMAT,
12711298
force: bool = False,
12721299
) -> list[tuple[Path, dict]]:
1273-
"""Generate one note file per lecture/video instead of a single merged file.
1300+
"""Generate one note file per lecture number instead of a single merged file.
12741301
1275-
Each video gets its own Markdown file: <course>_L<N>_notes.md
1302+
Lectures sharing the same lecture number (multi-part) are merged into
1303+
one file: <course>_L<N>_notes.md.
12761304
Returns a list of (path, scores) tuples for each generated note.
12771305
"""
1306+
from itertools import groupby
1307+
12781308
out_dir.mkdir(parents=True, exist_ok=True)
12791309
sections_dir = out_dir / "sections"
12801310
sections_dir.mkdir(exist_ok=True)
@@ -1286,62 +1316,70 @@ def generate_per_video_notes(
12861316

12871317
results: list[tuple[Path, dict]] = []
12881318

1289-
for ld in lectures:
1290-
lec_num = ld.num
1291-
lec_title = ld.title
1319+
for lec_num, ld_iter in groupby(lectures, key=lambda x: x.num):
1320+
ld_group = list(ld_iter)
1321+
multi_part = len(ld_group) > 1
12921322
ext = ".mdx" if fmt == "mdx" else ".md"
12931323
note_path = out_dir / f"{course_name}_L{lec_num:02d}_notes{ext}"
12941324

1295-
tqdm.write(f"\n ═══ Lecture {lec_num}: {lec_title} ═══")
1296-
1297-
n_chunks = max(1, (len(ld.slides) + CHAPTER_SIZE - 1) // CHAPTER_SIZE)
1298-
bar = tqdm(total=n_chunks, desc=f"L{lec_num}", unit="section",
1299-
bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]")
1300-
1301-
has_transcript = bool(ld.compact_by_idx)
1302-
chunks = [ld.slides[i:i + CHAPTER_SIZE]
1303-
for i in range(0, len(ld.slides), CHAPTER_SIZE)]
1304-
1305-
parts: list[str] = []
1325+
all_parts: list[str] = []
13061326
any_fresh = False
1307-
for ci, chunk in enumerate(chunks, start=1):
1308-
content, fresh = generate_section(
1309-
lec_num=lec_num,
1310-
lec_title=lec_title,
1311-
course_name=course_name,
1312-
chunk=chunk,
1313-
ci=ci,
1314-
ld=ld,
1315-
out_dir=out_dir,
1316-
sections_dir=sections_dir,
1317-
detail=detail,
1318-
has_transcript=has_transcript,
1319-
bar=bar,
1320-
force=force,
1321-
)
1322-
parts.append(content)
1323-
any_fresh = any_fresh or fresh
1324-
bar.update(1)
1325-
bar.close()
1326-
1327-
# Build per-video note with source info
1327+
source_lines: list[str] = []
1328+
1329+
for part_idx, ld in enumerate(ld_group, 1):
1330+
lec_title = ld.title
1331+
part_label = f" (Part {part_idx})" if multi_part else ""
1332+
tqdm.write(f"\n ═══ Lecture {lec_num}{part_label}: {lec_title} ═══")
1333+
1334+
n_chunks = max(1, (len(ld.slides) + CHAPTER_SIZE - 1) // CHAPTER_SIZE)
1335+
bar = tqdm(total=n_chunks, desc=f"L{lec_num}{part_label}",
1336+
unit="section",
1337+
bar_format="{l_bar}{bar}| {n_fmt}/{total_fmt} [{elapsed}<{remaining}]")
1338+
1339+
has_transcript = bool(ld.compact_by_idx)
1340+
chunks = [ld.slides[i:i + CHAPTER_SIZE]
1341+
for i in range(0, len(ld.slides), CHAPTER_SIZE)]
1342+
1343+
for ci, chunk in enumerate(chunks, start=1):
1344+
content, fresh = generate_section(
1345+
lec_num=lec_num,
1346+
lec_title=lec_title,
1347+
course_name=course_name,
1348+
chunk=chunk,
1349+
ci=ci,
1350+
ld=ld,
1351+
out_dir=out_dir,
1352+
sections_dir=sections_dir,
1353+
detail=detail,
1354+
has_transcript=has_transcript,
1355+
bar=bar,
1356+
force=force,
1357+
)
1358+
all_parts.append(content)
1359+
any_fresh = any_fresh or fresh
1360+
bar.update(1)
1361+
bar.close()
1362+
1363+
# Source metadata for this part
1364+
video_name = ld.compact.get("lecture", ld.slide_path.stem)
1365+
slide_file = ld.slide_path.name if ld.source != "screenshare" else "(screen recording frames)"
1366+
source_lines.append(f"- **Video**: {video_name} | **Slides**: {slide_file}")
1367+
1368+
# Build per-lecture note
13281369
from datetime import datetime
13291370
now = datetime.now().strftime("%Y-%m-%dT%H:%M:%S+08:00")
1330-
# Source metadata: video name and slide file
1331-
video_name = ld.compact.get("lecture", ld.slide_path.stem)
1332-
slide_file = ld.slide_path.name if ld.source != "screenshare" else "(screen recording frames)"
1333-
source_info = f"- **Video**: {video_name}\n- **Slides**: {slide_file}\n\n---\n\n"
1371+
source_info = "\n".join(source_lines) + "\n\n---\n\n"
13341372

13351373
front = (f"---\ntitle: {course_name} — Lecture {lec_num}\ndate: {now}\n"
13361374
f"description: Lecture notes generated by auto_note\n"
13371375
f"categories:\n - tech\n---\n\n")
1338-
heading = f"# {course_name} — Lecture {lec_num}: {lec_title}\n\n"
1339-
full_notes = front + heading + source_info + "\n\n".join(parts)
1376+
heading = f"# {course_name} — Lecture {lec_num}: {ld_group[0].title}\n\n"
1377+
full_notes = front + heading + source_info + "\n\n".join(all_parts)
13401378

13411379
# Image filter (only for newly generated content)
13421380
if any_fresh:
13431381
tqdm.write(" Running image filter pass…")
1344-
full_notes, _, _ = filter_images_pass(full_notes, out_dir, [ld])
1382+
full_notes, _, _ = filter_images_pass(full_notes, out_dir, ld_group)
13451383
else:
13461384
tqdm.write(" Skipping image filter (all sections from cache).")
13471385

@@ -1350,7 +1388,9 @@ def generate_per_video_notes(
13501388

13511389
scores = {}
13521390
if SHOW_SCORE:
1353-
scores = self_score(ld.slides, full_notes, ld.compact_slides)
1391+
all_slides = [s for ld in ld_group for s in ld.slides]
1392+
all_compact = [c for ld in ld_group for c in ld.compact_slides]
1393+
scores = self_score(all_slides, full_notes, all_compact)
13541394
_print_score(scores, note_path.name)
13551395
score_path = note_path.with_suffix(".score.json")
13561396
with open(score_path, "w") as f:

test/test_unit.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -382,10 +382,10 @@ def test_picks_short_label(self):
382382
]
383383
assert _chunk_title(slides) == "Process Management"
384384

385-
def test_fallback_to_first(self):
385+
def test_fallback_to_slide_range(self):
386386
from note_generation import _chunk_title, SlideInfo
387387
slides = [SlideInfo(0, "42", "...")]
388-
assert _chunk_title(slides) == "42"
388+
assert _chunk_title(slides) == "Slide 1" # bad label → falls back to slide number
389389

390390

391391
class TestImageRefPattern:

0 commit comments

Comments
 (0)