Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion schema/engine-version.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.34.0
0.35.0
21 changes: 4 additions & 17 deletions scripts/audit_content.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,6 @@
table so they can be fixed:

* duplicate cards within a lesson (same front, or same front/back pair)
* a free_text answer that is also listed as a distractor
* matching pairs / free_text accepts that don't line up with any
card the exercise references (possible wrong "correct" answer)
* empty / whitespace-only fields (card front/back, prompts, theory
body, titles, matching pair sides): the schema's ``minLength`` lets a
string of spaces through
Expand All @@ -19,9 +16,10 @@
and one rule lives in one place: malformed answer sets (cloze markers and
blanks, multiselect accept/distractors, word_tiles with fewer than two tiles,
picture_choice without exactly one correct image, a repeated matching left
term), duplicate card, step and exercise ids (learn-content-engine#202), and
the quality minimums such as matching pairs and free_text accepts (#185, keyed
to a lesson's ``purpose``). The distractor requirement on free_text and
term, a free_text answer that is also a distractor (#237, 0.35.0)),
duplicate card, step and exercise ids (learn-content-engine#202), and the
quality minimums such as matching pairs and free_text accepts (#185, keyed to
a lesson's ``purpose``). The distractor requirement on free_text and
picture_choice was dropped by the same decision.

Exit code is always 0 - this is advisory. ``--strict`` makes it exit 1
Expand Down Expand Up @@ -92,17 +90,6 @@ def add(problem, fix):
# against the card gloss - matching exercises legitimately pair
# a word with its article / gender / category, not its dictionary
# translation, so such a check is all false positives.
elif etype == "free_text":
accept = ex.get("accept") or []
# An accepted answer appearing verbatim in distractors is
# contradictory. Compare case-SENSITIVELY: a distractor that
# differs only by capitalisation (e.g. testing that Spanish
# nationalities are lower-case) is a legitimate wrong answer.
overlap = {a.strip() for a in accept} & {
d.strip() for d in (ex.get("distractors") or [])}
if overlap:
add(f"free_text '{eid}' accept & distractors overlap: {sorted(overlap)}",
"remove the overlap")


def main() -> int:
Expand Down
23 changes: 10 additions & 13 deletions tests/test_audit_content.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@
"""Tests for scripts/audit_content.py.

The audit is advisory and reports what no gate blocks: whitespace-only
fields, duplicate card fronts, a free_text answer that is also a
distractor, lessons missing from their set. What the engine gate blocks
(validate_with_engine.mjs) is not repeated here: a malformed cloze, a
multiselect overlap, too few word tiles, a picture choice without exactly
one correct image, a repeated matching left term. One rule, one place
(learn-content-engine's rule ownership).
fields, duplicate card fronts, lessons missing from their set. What the
engine gate blocks (validate_with_engine.mjs) is not repeated here: a
malformed cloze, a multiselect overlap, too few word tiles, a picture
choice without exactly one correct image, a repeated matching left term,
a free_text answer that is also a distractor (learn-content-engine 0.35.0).
One rule, one place (learn-content-engine's rule ownership).
"""
from __future__ import annotations

Expand Down Expand Up @@ -69,6 +69,10 @@ def _audit(exercise: dict, cards: list[dict] | None = None) -> list[str]:
"id": "m1", "type": "matching", "prompt": "Match.",
"pairs": [{"left": "a", "right": "1"}, {"left": "A", "right": "2"}, {"left": "b", "right": "3"}],
},
"free_text answer that is also a distractor (E-FREETEXT-DISJOINT)": {
"id": "f1", "type": "free_text", "prompt": "?",
"accept": ["gracias", "Gracias"], "distractors": ["gracias"],
},
}


Expand All @@ -90,13 +94,6 @@ def test_duplicate_card_front_is_reported() -> None:
assert any("duplicate card front" in problem for problem in problems)


def test_free_text_answer_that_is_also_a_distractor_is_reported() -> None:
problems = _audit(
{"id": "f1", "type": "free_text", "prompt": "?", "accept": ["gracias", "Gracias"], "distractors": ["gracias"]}
)
assert any("accept & distractors overlap" in problem for problem in problems)


def test_a_clean_lesson_reports_nothing() -> None:
cards = [{"id": "c1", "front": "Hund", "back": "dog"}]
exercise = {"id": "f1", "type": "free_text", "prompt": "Translate.", "accept": ["dog", "Dog"], "distractors": ["cat"]}
Expand Down
Loading