Skip to content

Commit cf28413

Browse files
committed
Drop the verifier/revision pass
The verifier was running a full extra LLM call against draft[:2500] after every chunk, asking gpt-4o to either say APPROVED or rewrite the whole excerpt. In practice it almost never caught a real terminology mistake on flagship note models (gpt-5.1, deepseek-v4-pro, claude-4.x); its main observable effect was the silent-truncation bug fixed in v1.0.4 — partial revisions overwriting good drafts. After the v1.0.4 guard, the verifier was already a no-op for any draft >2500 chars (most of them at detail=7). Net effect: paying a 5–15s gpt-4o call per chunk to produce something that gets discarded. Just remove it. Removed: • VERIFY_MODEL, VERIFY_NOTES constants • the "verify" prompt template • the post-draft verify block in generate_section • VERIFY_MODEL row from gui.py + Electron Settings dropdowns • verify-prompt artifact stripping in _clean_artifacts • run.py / gui.py / app.js fillGenInfo "Verifier" mentions The translator-chunking + auto-pick fixes from v1.0.4 stay — those addressed a real, recurring failure (Chinese translation hitting gpt-4o's 16K cap). The corresponding regression tests are renamed to TestVerifierRemoved and now assert the constants/prompt are gone.
1 parent 73d01a0 commit cf28413

6 files changed

Lines changed: 19 additions & 156 deletions

File tree

electron/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "auto-note",
3-
"version": "1.0.4",
3+
"version": "1.0.5",
44
"description": "AutoNote — lecture notes generator from Canvas recordings",
55
"homepage": "https://github.com/nodeeeeee/Auto-Note",
66
"author": {

electron/renderer/app.js

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1056,15 +1056,14 @@ async function loadLectureDropdown(cid) {
10561056
}
10571057

10581058
async function fillGenInfo() {
1059-
const [noteModel, verifyModel, target] = await Promise.all([
1059+
const [noteModel, target] = await Promise.all([
10601060
window.api.getConstant('generate', 'NOTE_MODEL'),
1061-
window.api.getConstant('generate', 'VERIFY_MODEL'),
10621061
window.api.getConstant('generate', 'QUALITY_TARGET'),
10631062
]);
10641063
const el = document.getElementById('gen-info-card');
10651064
if (el) {
10661065
el.innerHTML = mkCard(`<div class="info-row">${I.info}
1067-
<span>Generator: <strong>${esc(noteModel)}</strong> Verifier: <strong>${esc(verifyModel)}</strong>
1066+
<span>Generator: <strong>${esc(noteModel)}</strong>
10681067
Quality target: <strong>${esc(target)}</strong></span></div>`);
10691068
}
10701069
}
@@ -1223,21 +1222,6 @@ const CONSTANTS_DEF = [
12231222
// ── Claude CLI (uses `claude -p`, no API key needed) ────────────────
12241223
['Claude CLI (local)','claude-cli'],
12251224
]],
1226-
['generate', 'VERIFY_MODEL', 'Verification LLM', 'gpt-4.1-mini', [
1227-
// ── OpenAI ──────────────────────────────────────────────────────────
1228-
['GPT-4.1 mini ★','gpt-4.1-mini'],['GPT-4.1 nano','gpt-4.1-nano'],
1229-
['GPT-4.1','gpt-4.1'],['GPT-5.1','gpt-5.1'],['o4-mini (reasoning)','o4-mini'],
1230-
// ── Anthropic ───────────────────────────────────────────────────────
1231-
['Claude Haiku 4.5','claude-haiku-4-5-20251001'],['Claude Haiku 3.5','claude-3-5-haiku-20241022'],
1232-
['Claude Sonnet 4.6','claude-sonnet-4-6'],['Claude Sonnet 3.5','claude-3-5-sonnet-20241022'],
1233-
// ── Google Gemini ───────────────────────────────────────────────────
1234-
['Gemini 2.5 Flash','gemini-2.5-flash'],['Gemini 2.5 Flash Lite','gemini-2.5-flash-lite'],
1235-
['Gemini 2.0 Flash','gemini-2.0-flash'],
1236-
// ── DeepSeek ────────────────────────────────────────────────────────
1237-
['DeepSeek V4 Flash','deepseek-v4-flash'],['DeepSeek V4 Pro','deepseek-v4-pro'],
1238-
// ── xAI / Mistral ───────────────────────────────────────────────────
1239-
['Grok 3 mini','grok-3-mini'],['Mistral Small','mistral-small-latest'],
1240-
]],
12411225
['generate', 'DETAIL_LEVEL', 'Default detail level', '8', null],
12421226
['generate', 'CHAPTER_SIZE', 'Slides per GPT call', '15', null],
12431227
['generate', 'QUALITY_TARGET', 'Self-score target', '8.0', null],

gui.py

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1712,7 +1712,6 @@ def _on_course_select(e) -> None:
17121712
value=False, active_color=C_PRIMARY)
17131713

17141714
note_model = _read_constant("generate", "NOTE_MODEL")
1715-
verify_model = _read_constant("generate", "VERIFY_MODEL")
17161715
quality = _read_constant("generate", "QUALITY_TARGET")
17171716

17181717
def _run(_) -> None:
@@ -1750,8 +1749,7 @@ def _run(_) -> None:
17501749
_card(ft.Row(controls=[
17511750
ft.Icon(ft.Icons.INFO_OUTLINE, color=C_PRIMARY, size=15),
17521751
ft.Text(
1753-
f"Generator: {note_model} Verifier: {verify_model} "
1754-
f"Quality target: {quality}",
1752+
f"Generator: {note_model} Quality target: {quality}",
17551753
size=12,
17561754
color=ft.Colors.with_opacity(0.7, ft.Colors.WHITE),
17571755
),
@@ -2212,22 +2210,6 @@ def _worker():
22122210
("Claude Sonnet 4.5", "claude-sonnet-4-5"),
22132211
("Claude Haiku 4.5", "claude-haiku-4-5-20251001"),
22142212
]),
2215-
("generate", "VERIFY_MODEL", "Verification LLM", "gpt-4.1-mini", [
2216-
# OpenAI
2217-
("gpt-4.1-mini", "gpt-4.1-mini"),
2218-
("gpt-4.1-nano", "gpt-4.1-nano"),
2219-
("gpt-4.1", "gpt-4.1"),
2220-
("gpt-5.1", "gpt-5.1"),
2221-
# Gemini
2222-
("Gemini 3.5 Flash", "gemini-3.5-flash"),
2223-
("Gemini 3.0 Flash", "gemini-3.0-flash"),
2224-
("Gemini 2.5 Flash", "gemini-2.5-flash"),
2225-
("Gemini 2.0 Flash", "gemini-2.0-flash"),
2226-
# Anthropic
2227-
("Claude Haiku 4.5", "claude-haiku-4-5-20251001"),
2228-
("Claude Sonnet 4.5", "claude-sonnet-4-5"),
2229-
("Claude Sonnet 4.6", "claude-sonnet-4-6"),
2230-
]),
22312213
("generate", "DETAIL_LEVEL", "Default detail level", "8", None),
22322214
("generate", "CHAPTER_SIZE", "Slides per GPT call", "15", None),
22332215
("generate", "QUALITY_TARGET", "Self-score target", "8.0", None),

note_generation.py

Lines changed: 0 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,7 @@
6464
DETAIL_LEVEL = 7
6565
OUTPUT_FORMAT = "md"
6666
NOTE_MODEL = "gpt-5.1"
67-
VERIFY_MODEL = "gpt-4o"
6867
TRANSLATE_MODEL = "gpt-4o" # Chinese/other translation post-pass — gpt-4o is cheap enough
69-
VERIFY_NOTES = True
7068
QUALITY_TARGET = 8.0
7169
IMAGE_RENDER_SCALE = 1.5
7270
NOTE_LANGUAGE = "en" # "en" = English | "zh" = Chinese
@@ -156,18 +154,6 @@
156154
Format: `![Slide N](path) *(caption)*` or `![Frame N](path) *(caption)*`
157155
- Code examples must be complete and compilable, using the correct language tag (```c, ```cpp, ```python).
158156
""",
159-
verify="""\
160-
Check the following note excerpt for technical terminology consistency with the slides, and for any obvious factual errors.
161-
162-
**Reference glossary (from slides):**
163-
{term_list}
164-
165-
**Note excerpt:**
166-
{draft}
167-
168-
If there are no issues, reply APPROVED (this word only).
169-
If there are terminology or factual errors, return the corrected full note excerpt with no explanation.
170-
""",
171157
exam="""\
172158
Below are the complete lecture notes for {course_name}. Please append a concise exam cheat-sheet section at the end.
173159
@@ -845,14 +831,6 @@ def _clean_artifacts(text: str) -> str:
845831
cleaned = []
846832
for line in lines:
847833
stripped = line.strip()
848-
# Remove bare "APPROVED" lines (verifier leak)
849-
if stripped == "APPROVED":
850-
continue
851-
# Remove verify prompt leaks
852-
if "reply APPROVED (this word only)" in stripped:
853-
continue
854-
if "return the corrected full note excerpt" in stripped:
855-
continue
856834
if "terminology or factual errors" in stripped:
857835
continue
858836
# Clean section-header artifacts
@@ -1116,44 +1094,6 @@ def generate_section(
11161094
heading = f"### {lec_num}.{ci} {_chunk_title(chunk)}"
11171095
return f"{heading}\n\n*(Section could not be generated — re-run with force to retry.)*", True
11181096

1119-
if VERIFY_NOTES and draft:
1120-
terms = set()
1121-
for s in chunk:
1122-
for t in re.findall(r"\b[A-Z][a-zA-Z]{3,}\b|\b[A-Z]{3,}\b", s.text):
1123-
terms.add(t)
1124-
term_list = ", ".join(sorted(terms)[:30])
1125-
# The verifier only sees the head of the draft. If the draft is
1126-
# longer than that window, accepting v_result as the new draft
1127-
# would silently truncate everything past the window — exactly
1128-
# the bug that produced the ~2KB section files. Cap the window
1129-
# and refuse to overwrite drafts the verifier never fully saw.
1130-
VERIFY_INPUT_CAP = 2500
1131-
verifier_saw_all = len(draft) <= VERIFY_INPUT_CAP
1132-
v_user = _P("verify").format(term_list=term_list,
1133-
draft=draft[:VERIFY_INPUT_CAP])
1134-
# Always route the verify/revision pass through VERIFY_MODEL
1135-
# (gpt-4o) — cheap and fast, avoids burning codex quota on a
1136-
# short review call. If OpenAI is unavailable (no key, quota
1137-
# exceeded, network issue), skip verification rather than losing
1138-
# the generated draft.
1139-
_vmodel = VERIFY_MODEL
1140-
try:
1141-
v_result = _call(_vmodel, "", v_user, 1500)
1142-
if not v_result.strip().upper().startswith("APPROVED"):
1143-
if not verifier_saw_all:
1144-
# Long draft — refuse to overwrite with a verifier
1145-
# revision that only saw the first 2500 chars.
1146-
tqdm.write(f" [warn] Verifier flagged issues but draft "
1147-
f"({len(draft)} chars) exceeds verify window — "
1148-
f"keeping full draft as-is.")
1149-
elif len(v_result) > len(draft) * 0.5:
1150-
draft = v_result
1151-
else:
1152-
tqdm.write(f" [warn] Verifier suspicious response, keeping draft")
1153-
except Exception as _ve:
1154-
tqdm.write(f" [warn] Verify pass failed ({type(_ve).__name__}: "
1155-
f"{str(_ve)[:120]}) — keeping draft")
1156-
11571097
# Strip pipeline artifacts that may have leaked into the draft
11581098
draft = _clean_artifacts(draft)
11591099

run.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -694,7 +694,6 @@ def _const(script: str, name: str, desc: str) -> None:
694694
_const("align", "OFF_SLIDE_THRESHOLD", "Min cosine to stay on-slide")
695695
_const("align", "PRIOR_SIGMA", "Temporal prior width (slides)")
696696
_const("generate", "NOTE_MODEL", "LLM for note generation")
697-
_const("generate", "VERIFY_MODEL", "LLM for verification")
698697
_const("generate", "DETAIL_LEVEL", "Default detail level (0-10)")
699698
_const("generate", "CHAPTER_SIZE", "Slides per GPT call")
700699
_const("generate", "QUALITY_TARGET", "Self-score target for --iterate")

test/test_truncation_fixes.py

Lines changed: 15 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -85,64 +85,22 @@ def test_passes_through_cli_models(self, monkeypatch):
8585
assert ng._pick_translate_model() == "codex-cli"
8686

8787

88-
class TestVerifyOverwriteGuard:
89-
"""The verifier sees only ``draft[:2500]``. When the draft is longer
90-
than that, accepting v_result as the new draft truncates everything
91-
past the verify window. The new behavior preserves the full draft
92-
and only allows the verifier to overwrite when it saw all of it.
88+
class TestVerifierRemoved:
89+
"""The verifier/revision pass was removed in v1.0.5 — modern flagship
90+
note models rarely make terminology mistakes worth a separate review
91+
call, and the limited verify window had been silently truncating
92+
long drafts. Make sure no constant or prompt template is left over.
9393
"""
9494

95-
def test_long_draft_kept_intact_when_verifier_disagrees(self, monkeypatch):
96-
# Construct a draft longer than VERIFY_INPUT_CAP=2500. The mock
97-
# verifier returns a "revised" version that is shorter — under
98-
# the old logic it would replace the draft and silently truncate
99-
# the tail. Under the fix, the original draft is kept.
95+
def test_constants_gone(self):
10096
import note_generation as ng
97+
assert not hasattr(ng, "VERIFY_NOTES")
98+
assert not hasattr(ng, "VERIFY_MODEL")
99+
100+
def test_verify_prompt_not_in_template(self):
101+
import note_generation as ng
102+
# Both language tables should have lost the "verify" key
103+
assert "verify" not in ng._PROMPTS["en"]
104+
if "zh" in ng._PROMPTS:
105+
assert "verify" not in ng._PROMPTS["zh"]
101106

102-
long_draft = "Sentence about TCP. " * 200 # ~3800 chars
103-
revised = "TCP is a protocol. " * 50 # ~950 chars
104-
105-
captured = {"draft": long_draft}
106-
107-
def fake_call(model, system, user, max_tokens, _truncated=None):
108-
# Verifier path
109-
if "Reference glossary" in user or "Reference Glossary" in user:
110-
return revised
111-
# Translator path (skip — language is en in this test)
112-
return captured["draft"]
113-
114-
# Build the minimal context generate_section needs, then assert
115-
# the draft did not collapse to ~revised. Easier: just exercise
116-
# the guard logic directly — patch _call and call generate_section
117-
# would require a full LectureData mock. Instead, replicate the
118-
# post-verify fragment here:
119-
VERIFY_INPUT_CAP = 2500
120-
draft = long_draft
121-
verifier_saw_all = len(draft) <= VERIFY_INPUT_CAP
122-
v_result = revised
123-
124-
if not v_result.strip().upper().startswith("APPROVED"):
125-
if not verifier_saw_all:
126-
# The fix: refuse to overwrite
127-
pass
128-
elif len(v_result) > len(draft) * 0.5:
129-
draft = v_result
130-
131-
assert draft == long_draft, (
132-
"Long draft must NOT be replaced by a partial verifier revision"
133-
)
134-
135-
def test_short_draft_can_be_replaced_by_verifier(self):
136-
# When the verifier saw the entire draft, replacement is safe.
137-
VERIFY_INPUT_CAP = 2500
138-
draft = "Short draft about UDP." * 5 # ~110 chars
139-
v_result = "UDP is a connectionless protocol." * 5 # ~165 chars
140-
141-
verifier_saw_all = len(draft) <= VERIFY_INPUT_CAP
142-
if not v_result.strip().upper().startswith("APPROVED"):
143-
if not verifier_saw_all:
144-
pass
145-
elif len(v_result) > len(draft) * 0.5:
146-
draft = v_result
147-
148-
assert draft == v_result

0 commit comments

Comments
 (0)