diff --git a/scripts/audit_content.py b/scripts/audit_content.py index f326b1c..d697e71 100644 --- a/scripts/audit_content.py +++ b/scripts/audit_content.py @@ -7,17 +7,19 @@ table so they can be fixed: * duplicate cards within a lesson (same front, or same front/back pair) - * exercises whose answer set is malformed for their type - (matching pairs, free_text accept/distractors overlap, cloze blanks, - word_tiles tiles, picture_choice single correct) + * a free_text answer that is also listed as a distractor * matching pairs / free_text accepts that don't line up with any card the exercise references (possible wrong "correct" answer) * empty / whitespace-only fields (card front/back, prompts, theory - body, titles) + body, titles, matching pair sides): the schema's ``minLength`` lets a + string of spaces through * lessons missing in their set manifest, or set fields missing -Not here any more, because the engine gate (validate_with_engine.mjs) blocks -on them: duplicate card, step and exercise ids (learn-content-engine#202), and +Not here, because the engine gate (validate_with_engine.mjs) blocks on them, +and one rule lives in one place: malformed answer sets (cloze markers and +blanks, multiselect accept/distractors, word_tiles with fewer than two tiles, +picture_choice without exactly one correct image, a repeated matching left +term), duplicate card, step and exercise ids (learn-content-engine#202), and the quality minimums such as matching pairs and free_text accepts (#185, keyed to a lesson's ``purpose``). The distractor requirement on free_text and picture_choice was dropped by the same decision. @@ -83,16 +85,9 @@ def add(problem, fix): add(f"exercise '{eid}' ({etype}) has empty prompt", "add a prompt") if etype == "matching": - pairs = ex.get("pairs") or [] - seen_left = set() - for p in pairs: - left = (p.get("left") or "").strip() - right = (p.get("right") or "").strip() + for p in ex.get("pairs") or []: if is_blank(p.get("left")) or is_blank(p.get("right")): add(f"matching '{eid}' has an empty pair side", "fill left/right") - if left.lower() in seen_left: - add(f"matching '{eid}' duplicate left '{left}'", "remove duplicate pair") - seen_left.add(left.lower()) # NB: we deliberately do NOT cross-check the pair's right side # against the card gloss - matching exercises legitimately pair # a word with its article / gender / category, not its dictionary @@ -108,40 +103,6 @@ def add(problem, fix): if overlap: add(f"free_text '{eid}' accept & distractors overlap: {sorted(overlap)}", "remove the overlap") - elif etype == "cloze": - # cloze has three modes. "multiselect" ("select all that apply") - # deliberately carries NO ___ markers and NO blanks: the sentence - # is the question stem, and accept/distractors hold the options. - # Auditing it against the type/select marker+blanks shape produces - # only false positives, so branch on cloze_mode (mirrors the - # engine's per-mode semantics; validate_content.py already does). - if (ex.get("cloze_mode") or "type") == "multiselect": - accept = {a.strip() for a in (ex.get("accept") or []) if not is_blank(a)} - distractors = {d.strip() for d in (ex.get("distractors") or []) if not is_blank(d)} - if not accept: - add(f"cloze '{eid}' (multiselect) has no accepted options", "add accept options") - if not distractors: - add(f"cloze '{eid}' (multiselect) has no distractors", "add distractors") - overlap = accept & distractors - if overlap: - add(f"cloze '{eid}' (multiselect) accept & distractors overlap: {sorted(overlap)}", - "remove the overlap") - else: - blanks = ex.get("blanks") or [] - sentence = ex.get("sentence") or "" - if "___" not in sentence: - add(f"cloze '{eid}' sentence has no ___ gap", "add a ___ gap") - if not blanks or any(not (b.get("accept") or []) for b in blanks): - add(f"cloze '{eid}' has a blank with no accepted answers", "add accepts") - elif etype == "word_tiles": - if len(ex.get("tiles") or []) < 2: - add(f"word_tiles '{eid}' has < 2 tiles", "add tiles") - elif etype == "picture_choice": - images = ex.get("images") or [] - correct = [i for i in images if str(i.get("is_correct")).lower() == "true"] - if len(correct) != 1: - add(f"picture_choice '{eid}' has {len(correct)} correct images (need 1)", - "mark exactly one is_correct") def main() -> int: diff --git a/tests/test_audit_content.py b/tests/test_audit_content.py index aee976d..fc6574e 100644 --- a/tests/test_audit_content.py +++ b/tests/test_audit_content.py @@ -1,15 +1,21 @@ #!/usr/bin/env python3 """Tests for scripts/audit_content.py. -Focus: the cloze-mode branch. `multiselect` cloze has no ___ markers and -no blanks by design, so the audit must NOT flag it for missing markers / -blanks (issue #1). `type`/`select` cloze keeps the marker+blanks checks. +The audit is advisory and reports what no gate blocks: whitespace-only +fields, duplicate card fronts, a free_text answer that is also a +distractor, lessons missing from their set. What the engine gate blocks +(validate_with_engine.mjs) is not repeated here: a malformed cloze, a +multiselect overlap, too few word tiles, a picture choice without exactly +one correct image, a repeated matching left term. One rule, one place +(learn-content-engine's rule ownership). """ from __future__ import annotations import sys from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[1] SCRIPTS_DIR = REPO_ROOT / "scripts" if str(SCRIPTS_DIR) not in sys.path: @@ -18,100 +24,80 @@ import audit_content as ac # noqa: E402 -def _audit(exercise: dict) -> list[str]: - """Run the audit over a one-exercise lesson; return the problem strings.""" - lesson = { - "id": "l1", - "title": "T", - "steps": [{"id": "s1", "type": "exercise", "exercise": exercise}], - } +def _audit_lesson(lesson: dict) -> list[str]: findings: list[tuple] = [] ac.audit_lesson(lesson, "", findings) return [problem for _label, problem, _fix in findings] -def _cloze_findings(exercise: dict) -> list[str]: - return [p for p in _audit(exercise) if "cloze" in p] - - -def test_valid_multiselect_is_not_flagged(): - """A correct multiselect (no markers, no blanks) produces no cloze finding.""" - ex = { - "id": "c1", - "type": "cloze", - "cloze_mode": "multiselect", - "prompt": "Select all that apply.", - "sentence": "Which of these are prime numbers?", - "accept": ["2", "3", "5"], - "distractors": ["4", "6"], - } - assert _cloze_findings(ex) == [] - - -def test_multiselect_overlap_is_flagged(): - ex = { - "id": "c1", - "type": "cloze", - "cloze_mode": "multiselect", - "prompt": "Select all that apply.", - "sentence": "Which are primes?", - "accept": ["2", "3"], - "distractors": ["3", "4"], # "3" overlaps - } - findings = _cloze_findings(ex) - assert any("overlap" in f for f in findings) - - -def test_multiselect_without_distractors_is_flagged(): - ex = { - "id": "c1", - "type": "cloze", - "cloze_mode": "multiselect", - "prompt": "Select all that apply.", - "sentence": "Which are primes?", - "accept": ["2", "3"], - "distractors": [], - } - findings = _cloze_findings(ex) - assert any("no distractors" in f for f in findings) - - -def test_type_cloze_missing_marker_still_flagged(): - """Regression: the type/select path keeps its marker + blanks checks.""" - ex = { - "id": "c1", - "type": "cloze", - "cloze_mode": "type", - "prompt": "Fill in.", - "sentence": "no gap here", # missing ___ - "blanks": [{"accept": ["x"]}], - } - findings = _cloze_findings(ex) - assert any("no ___ gap" in f for f in findings) - - -def test_select_cloze_empty_blank_still_flagged(): - ex = { - "id": "c1", - "type": "cloze", - "cloze_mode": "select", - "prompt": "Choose.", - "sentence": "a ___ b", - "blanks": [{"accept": []}], # empty accept - "distractors": ["x"], - } - findings = _cloze_findings(ex) - assert any("no accepted answers" in f for f in findings) - - -def test_default_mode_is_type_marker_checked(): - """No cloze_mode defaults to type, so the marker check applies.""" - ex = { - "id": "c1", - "type": "cloze", - "prompt": "Fill in.", - "sentence": "no gap", - "blanks": [{"accept": ["x"]}], - } - findings = _cloze_findings(ex) - assert any("no ___ gap" in f for f in findings) +def _audit(exercise: dict, cards: list[dict] | None = None) -> list[str]: + """Run the audit over a one-exercise lesson; return the problem strings.""" + return _audit_lesson( + { + "id": "l1", + "title": "T", + "cards": cards or [], + "steps": [{"id": "s1", "type": "exercise", "exercise": exercise}], + } + ) + + +ENGINE_OWNED = { + "multiselect accept/distractors overlap (E-CLOZE-MS-DISJOINT)": { + "id": "c1", "type": "cloze", "cloze_mode": "multiselect", "prompt": "Pick.", + "sentence": "Which are prime?", "accept": ["2", "3"], "distractors": ["3", "4"], + }, + "multiselect without distractors (E-CLOZE-MS-DISTRACTORS)": { + "id": "c1", "type": "cloze", "cloze_mode": "multiselect", "prompt": "Pick.", + "sentence": "Which are prime?", "accept": ["2"], "distractors": [], + }, + "type cloze without a marker (E-CLOZE-MARKERS)": { + "id": "c1", "type": "cloze", "cloze_mode": "type", "prompt": "Fill.", + "sentence": "No gap here.", "blanks": [{"accept": ["x"]}], + }, + "word_tiles with one tile (E-TILES-MIN)": { + "id": "w1", "type": "word_tiles", "prompt": "Order.", "tiles": ["one"], + }, + "picture_choice with two correct images (E-PIC-ONE-CORRECT)": { + "id": "p1", "type": "picture_choice", "prompt": "Which?", + "images": [ + {"src": "a.png", "label": "a", "is_correct": "true"}, + {"src": "b.png", "label": "b", "is_correct": "true"}, + ], + }, + "matching with a repeated left term (E-MATCH-DUP-LEFT)": { + "id": "m1", "type": "matching", "prompt": "Match.", + "pairs": [{"left": "a", "right": "1"}, {"left": "A", "right": "2"}, {"left": "b", "right": "3"}], + }, +} + + +@pytest.mark.parametrize("label", sorted(ENGINE_OWNED)) +def test_engine_owned_errors_are_not_repeated(label: str) -> None: + """The engine gate blocks these; the audit does not report them again.""" + assert _audit(ENGINE_OWNED[label]) == [] + + +def test_whitespace_only_prompt_is_reported() -> None: + """The schema's minLength lets " " through; the audit catches it.""" + problems = _audit({"id": "f1", "type": "free_text", "prompt": " ", "accept": ["a", "b"]}) + assert any("empty prompt" in problem for problem in problems) + + +def test_duplicate_card_front_is_reported() -> None: + cards = [{"id": "c1", "front": "Hund", "back": "dog"}, {"id": "c2", "front": "hund", "back": "hound"}] + problems = _audit({"id": "f1", "type": "free_text", "prompt": "?", "accept": ["a", "b"]}, cards) + assert any("duplicate card front" in problem for problem in problems) + + +def test_free_text_answer_that_is_also_a_distractor_is_reported() -> None: + problems = _audit( + {"id": "f1", "type": "free_text", "prompt": "?", "accept": ["gracias", "Gracias"], "distractors": ["gracias"]} + ) + assert any("accept & distractors overlap" in problem for problem in problems) + + +def test_a_clean_lesson_reports_nothing() -> None: + cards = [{"id": "c1", "front": "Hund", "back": "dog"}] + exercise = {"id": "f1", "type": "free_text", "prompt": "Translate.", "accept": ["dog", "Dog"], "distractors": ["cat"]} + assert _audit(exercise, cards) == []