Skip to content

chore(tooling): the audit stops repeating errors the engine gate blocks #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We鈥檒l occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
astrapi69 merged 1 commit into from
Oct 1, 2026
Merged

chore(tooling): the audit stops repeating errors the engine gate blocks #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 9 additions & 48 deletions scripts/audit_content.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,19 @@
table so they can be fixed:

* duplicate cards within a lesson (same front, or same front/back pair)
* exercises whose answer set is malformed for their type
(matching pairs, free_text accept/distractors overlap, cloze blanks,
word_tiles tiles, picture_choice single correct)
* a free_text answer that is also listed as a distractor
* matching pairs / free_text accepts that don't line up with any
card the exercise references (possible wrong "correct" answer)
* empty / whitespace-only fields (card front/back, prompts, theory
body, titles)
body, titles, matching pair sides): the schema's ``minLength`` lets a
string of spaces through
* lessons missing in their set manifest, or set fields missing

Not here any more, because the engine gate (validate_with_engine.mjs) blocks
on them: duplicate card, step and exercise ids (learn-content-engine#202), and
Not here, because the engine gate (validate_with_engine.mjs) blocks on them,
and one rule lives in one place: malformed answer sets (cloze markers and
blanks, multiselect accept/distractors, word_tiles with fewer than two tiles,
picture_choice without exactly one correct image, a repeated matching left
term), duplicate card, step and exercise ids (learn-content-engine#202), and
the quality minimums such as matching pairs and free_text accepts (#185, keyed
to a lesson's ``purpose``). The distractor requirement on free_text and
picture_choice was dropped by the same decision.
Expand Down Expand Up @@ -83,16 +85,9 @@ def add(problem, fix):
add(f"exercise '{eid}' ({etype}) has empty prompt", "add a prompt")

if etype == "matching":
pairs = ex.get("pairs") or []
seen_left = set()
for p in pairs:
left = (p.get("left") or "").strip()
right = (p.get("right") or "").strip()
for p in ex.get("pairs") or []:
if is_blank(p.get("left")) or is_blank(p.get("right")):
add(f"matching '{eid}' has an empty pair side", "fill left/right")
if left.lower() in seen_left:
add(f"matching '{eid}' duplicate left '{left}'", "remove duplicate pair")
seen_left.add(left.lower())
# NB: we deliberately do NOT cross-check the pair's right side
# against the card gloss - matching exercises legitimately pair
# a word with its article / gender / category, not its dictionary
Expand All @@ -108,40 +103,6 @@ def add(problem, fix):
if overlap:
add(f"free_text '{eid}' accept & distractors overlap: {sorted(overlap)}",
"remove the overlap")
elif etype == "cloze":
# cloze has three modes. "multiselect" ("select all that apply")
# deliberately carries NO ___ markers and NO blanks: the sentence
# is the question stem, and accept/distractors hold the options.
# Auditing it against the type/select marker+blanks shape produces
# only false positives, so branch on cloze_mode (mirrors the
# engine's per-mode semantics; validate_content.py already does).
if (ex.get("cloze_mode") or "type") == "multiselect":
accept = {a.strip() for a in (ex.get("accept") or []) if not is_blank(a)}
distractors = {d.strip() for d in (ex.get("distractors") or []) if not is_blank(d)}
if not accept:
add(f"cloze '{eid}' (multiselect) has no accepted options", "add accept options")
if not distractors:
add(f"cloze '{eid}' (multiselect) has no distractors", "add distractors")
overlap = accept & distractors
if overlap:
add(f"cloze '{eid}' (multiselect) accept & distractors overlap: {sorted(overlap)}",
"remove the overlap")
else:
blanks = ex.get("blanks") or []
sentence = ex.get("sentence") or ""
if "___" not in sentence:
add(f"cloze '{eid}' sentence has no ___ gap", "add a ___ gap")
if not blanks or any(not (b.get("accept") or []) for b in blanks):
add(f"cloze '{eid}' has a blank with no accepted answers", "add accepts")
elif etype == "word_tiles":
if len(ex.get("tiles") or []) < 2:
add(f"word_tiles '{eid}' has < 2 tiles", "add tiles")
elif etype == "picture_choice":
images = ex.get("images") or []
correct = [i for i in images if str(i.get("is_correct")).lower() == "true"]
if len(correct) != 1:
add(f"picture_choice '{eid}' has {len(correct)} correct images (need 1)",
"mark exactly one is_correct")


def main() -> int:
Expand Down
176 changes: 81 additions & 95 deletions tests/test_audit_content.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,21 @@
#!/usr/bin/env python3
"""Tests for scripts/audit_content.py.

Focus: the cloze-mode branch. `multiselect` cloze has no ___ markers and
no blanks by design, so the audit must NOT flag it for missing markers /
blanks (issue #1). `type`/`select` cloze keeps the marker+blanks checks.
The audit is advisory and reports what no gate blocks: whitespace-only
fields, duplicate card fronts, a free_text answer that is also a
distractor, lessons missing from their set. What the engine gate blocks
(validate_with_engine.mjs) is not repeated here: a malformed cloze, a
multiselect overlap, too few word tiles, a picture choice without exactly
one correct image, a repeated matching left term. One rule, one place
(learn-content-engine's rule ownership).
"""
from __future__ import annotations

import sys
from pathlib import Path

import pytest

REPO_ROOT = Path(__file__).resolve().parents[1]
SCRIPTS_DIR = REPO_ROOT / "scripts"
if str(SCRIPTS_DIR) not in sys.path:
Expand All @@ -18,100 +24,80 @@
import audit_content as ac # noqa: E402


def _audit(exercise: dict) -> list[str]:
"""Run the audit over a one-exercise lesson; return the problem strings."""
lesson = {
"id": "l1",
"title": "T",
"steps": [{"id": "s1", "type": "exercise", "exercise": exercise}],
}
def _audit_lesson(lesson: dict) -> list[str]:
findings: list[tuple] = []
ac.audit_lesson(lesson, "<lesson>", findings)
return [problem for _label, problem, _fix in findings]


def _cloze_findings(exercise: dict) -> list[str]:
return [p for p in _audit(exercise) if "cloze" in p]


def test_valid_multiselect_is_not_flagged():
"""A correct multiselect (no markers, no blanks) produces no cloze finding."""
ex = {
"id": "c1",
"type": "cloze",
"cloze_mode": "multiselect",
"prompt": "Select all that apply.",
"sentence": "Which of these are prime numbers?",
"accept": ["2", "3", "5"],
"distractors": ["4", "6"],
}
assert _cloze_findings(ex) == []


def test_multiselect_overlap_is_flagged():
ex = {
"id": "c1",
"type": "cloze",
"cloze_mode": "multiselect",
"prompt": "Select all that apply.",
"sentence": "Which are primes?",
"accept": ["2", "3"],
"distractors": ["3", "4"], # "3" overlaps
}
findings = _cloze_findings(ex)
assert any("overlap" in f for f in findings)


def test_multiselect_without_distractors_is_flagged():
ex = {
"id": "c1",
"type": "cloze",
"cloze_mode": "multiselect",
"prompt": "Select all that apply.",
"sentence": "Which are primes?",
"accept": ["2", "3"],
"distractors": [],
}
findings = _cloze_findings(ex)
assert any("no distractors" in f for f in findings)


def test_type_cloze_missing_marker_still_flagged():
"""Regression: the type/select path keeps its marker + blanks checks."""
ex = {
"id": "c1",
"type": "cloze",
"cloze_mode": "type",
"prompt": "Fill in.",
"sentence": "no gap here", # missing ___
"blanks": [{"accept": ["x"]}],
}
findings = _cloze_findings(ex)
assert any("no ___ gap" in f for f in findings)


def test_select_cloze_empty_blank_still_flagged():
ex = {
"id": "c1",
"type": "cloze",
"cloze_mode": "select",
"prompt": "Choose.",
"sentence": "a ___ b",
"blanks": [{"accept": []}], # empty accept
"distractors": ["x"],
}
findings = _cloze_findings(ex)
assert any("no accepted answers" in f for f in findings)


def test_default_mode_is_type_marker_checked():
"""No cloze_mode defaults to type, so the marker check applies."""
ex = {
"id": "c1",
"type": "cloze",
"prompt": "Fill in.",
"sentence": "no gap",
"blanks": [{"accept": ["x"]}],
}
findings = _cloze_findings(ex)
assert any("no ___ gap" in f for f in findings)
def _audit(exercise: dict, cards: list[dict] | None = None) -> list[str]:
"""Run the audit over a one-exercise lesson; return the problem strings."""
return _audit_lesson(
{
"id": "l1",
"title": "T",
"cards": cards or [],
"steps": [{"id": "s1", "type": "exercise", "exercise": exercise}],
}
)


ENGINE_OWNED = {
"multiselect accept/distractors overlap (E-CLOZE-MS-DISJOINT)": {
"id": "c1", "type": "cloze", "cloze_mode": "multiselect", "prompt": "Pick.",
"sentence": "Which are prime?", "accept": ["2", "3"], "distractors": ["3", "4"],
},
"multiselect without distractors (E-CLOZE-MS-DISTRACTORS)": {
"id": "c1", "type": "cloze", "cloze_mode": "multiselect", "prompt": "Pick.",
"sentence": "Which are prime?", "accept": ["2"], "distractors": [],
},
"type cloze without a marker (E-CLOZE-MARKERS)": {
"id": "c1", "type": "cloze", "cloze_mode": "type", "prompt": "Fill.",
"sentence": "No gap here.", "blanks": [{"accept": ["x"]}],
},
"word_tiles with one tile (E-TILES-MIN)": {
"id": "w1", "type": "word_tiles", "prompt": "Order.", "tiles": ["one"],
},
"picture_choice with two correct images (E-PIC-ONE-CORRECT)": {
"id": "p1", "type": "picture_choice", "prompt": "Which?",
"images": [
{"src": "a.png", "label": "a", "is_correct": "true"},
{"src": "b.png", "label": "b", "is_correct": "true"},
],
},
"matching with a repeated left term (E-MATCH-DUP-LEFT)": {
"id": "m1", "type": "matching", "prompt": "Match.",
"pairs": [{"left": "a", "right": "1"}, {"left": "A", "right": "2"}, {"left": "b", "right": "3"}],
},
}


@pytest.mark.parametrize("label", sorted(ENGINE_OWNED))
def test_engine_owned_errors_are_not_repeated(label: str) -> None:
"""The engine gate blocks these; the audit does not report them again."""
assert _audit(ENGINE_OWNED[label]) == []


def test_whitespace_only_prompt_is_reported() -> None:
"""The schema's minLength lets " " through; the audit catches it."""
problems = _audit({"id": "f1", "type": "free_text", "prompt": " ", "accept": ["a", "b"]})
assert any("empty prompt" in problem for problem in problems)


def test_duplicate_card_front_is_reported() -> None:
cards = [{"id": "c1", "front": "Hund", "back": "dog"}, {"id": "c2", "front": "hund", "back": "hound"}]
problems = _audit({"id": "f1", "type": "free_text", "prompt": "?", "accept": ["a", "b"]}, cards)
assert any("duplicate card front" in problem for problem in problems)


def test_free_text_answer_that_is_also_a_distractor_is_reported() -> None:
problems = _audit(
{"id": "f1", "type": "free_text", "prompt": "?", "accept": ["gracias", "Gracias"], "distractors": ["gracias"]}
)
assert any("accept & distractors overlap" in problem for problem in problems)


def test_a_clean_lesson_reports_nothing() -> None:
cards = [{"id": "c1", "front": "Hund", "back": "dog"}]
exercise = {"id": "f1", "type": "free_text", "prompt": "Translate.", "accept": ["dog", "Dog"], "distractors": ["cat"]}
assert _audit(exercise, cards) == []
Loading