From 07903d0dc65380f01e4729f7410fca98a2bc38e9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 00:59:26 +0000 Subject: [PATCH 1/4] Reuse source inventory while checking present translations Signed-off-by: zlareb1 --- scripts/i18n/translate.py | 49 ++-- tests/unit/test_i18n_present_inventory.py | 282 ++++++++++++++++++++++ 2 files changed, 314 insertions(+), 17 deletions(-) create mode 100644 tests/unit/test_i18n_present_inventory.py diff --git a/scripts/i18n/translate.py b/scripts/i18n/translate.py index b8c236bb32..a4a08efdd5 100644 --- a/scripts/i18n/translate.py +++ b/scripts/i18n/translate.py @@ -277,9 +277,13 @@ def link_routes(link: str, image: bool, source: str, lang: str) -> tuple[str, .. return (english, native) -def rewrite_links(text: str, source: str, lang: str) -> str: - """Point every relative link of `source` at its target as seen from the translation.""" - inventory = frozenset(sources()) +def rewrite_links( + text: str, source: str, lang: str, *, inventory: frozenset[str] | None = None +) -> str: + """Point relative links at their translation targets, checking live paths and anchors + even when tracked-name membership comes from the caller's pass.""" + if inventory is None: + inventory = frozenset(sources()) edits = [] for start, end, link, image in link_targets(text): routes = link_routes(link, image, source, lang) @@ -287,14 +291,15 @@ def rewrite_links(text: str, source: str, lang: str) -> str: if len(routes) == 2: path, _, anchor = link.partition("#") resolved = posixpath.normpath(posixpath.join(posixpath.dirname(source), unquote(path))) - translated = ROOT / target_path(lang, resolved) - if ( - resolved in inventory - and translated.is_file() - and not translated.is_symlink() - and (not anchor or f'' in translated.read_text()) - ): - target = routes[1] + translated_path = target_path(lang, resolved) + translated = ROOT / translated_path + if resolved in inventory: + if not lifecycle.regular(resolved): + raise SystemExit(f"English source is not a regular file: {resolved}") + if lifecycle.regular(translated_path) and ( + not anchor or f'' in translated.read_text() + ): + target = routes[1] edits.append((start, end, target)) return replace_spans(text, edits) @@ -611,19 +616,23 @@ def link_problems() -> list[str]: def untranslated_pages() -> list[str]: """Translations that kept a heading, link text, alt text or table cell in English.""" problems = [] - for source in sources(): + discovered = sources() + inventory = frozenset(discovered) + for source in discovered: for lang in LANGUAGES: - path = ROOT / target_path(lang.code, source) + rel = target_path(lang.code, source) + path = ROOT / rel if not path.is_file(): continue - english = "\n".join(english_requests(prepare(source, lang)[0])) + if not lifecycle.regular(rel): + raise SystemExit(f"{rel}: not a regular generated Markdown page") + english = "\n".join(english_requests(prepare(source, lang, inventory=inventory)[0])) text = path.read_text() # Generated banner links must not shift the source body's node identities. if (stamp := STAMP.match(text)) and stamp[1] == source: prefix = header(source, lang, stamp[3], stamp[2]) if text.startswith(prefix): text = text[len(prefix) :] - rel = path.relative_to(ROOT).as_posix() english = lifecycle.canonical_links(english, rel) text = lifecycle.canonical_links(text, rel) problems.extend( @@ -828,15 +837,21 @@ def language_placeholder(text: str) -> list[tuple[int, int, str]]: return [(match.start(), match.end(), "")] if match else [] -def prepare(source: str, lang: Language) -> tuple[str, str]: +def prepare( + source: str, lang: Language, *, inventory: frozenset[str] | None = None +) -> tuple[str, str]: """The English text to translate and the language line to put back afterwards ('' when the page has none).""" + if inventory is None: + inventory = frozenset(sources()) + if not lifecycle.regular(source): + raise SystemExit(f"English source is not a regular file: {source}") text = (ROOT / source).read_text() target = target_path(lang.code, source) edits = language_placeholder(text) line = language_line(target) if edits else "" text = replace_spans(text, edits) - return rewrite_links(text, source, lang.code), line + return rewrite_links(text, source, lang.code, inventory=inventory), line def render(source: str, lang: Language, body: str, line: str, model: str) -> str: diff --git a/tests/unit/test_i18n_present_inventory.py b/tests/unit/test_i18n_present_inventory.py new file mode 100644 index 0000000000..3902ff0754 --- /dev/null +++ b/tests/unit/test_i18n_present_inventory.py @@ -0,0 +1,282 @@ +"""Present-output validation shares discovery and checks each live file at use.""" + +from __future__ import annotations + +import pathlib +import subprocess + +import pytest + +from scripts.i18n import translate as t + +JA = t.LANGUAGES[1] + + +def git(root: pathlib.Path, *args: str) -> None: + subprocess.run(["git", *args], cwd=root, check=True, capture_output=True) + + +def publish(root: pathlib.Path, source: str) -> pathlib.Path: + prepared, line = t.prepare(source, JA) + body = "\n".join(t.english_requests(prepared)) + for english, japanese in (("A", "一"), ("B", "二"), ("C", "三"), ("Added", "追加")): + body = body.replace("# " + english + "\n", "# " + japanese + "\n") + output = root / t.target_path(JA.code, source) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(t.render(source, JA, body, line, t.DEFAULT_MODEL)) + return output + + +@pytest.fixture +def repo(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> pathlib.Path: + git(tmp_path, "init", "-q", "-b", "main") + (tmp_path / "docs").mkdir() + for name in ("a", "b"): + (tmp_path / f"docs/{name}.md").write_text(f"# {name.upper()}\n") + git(tmp_path, "add", "docs") + monkeypatch.setattr(t, "ROOT", tmp_path) + monkeypatch.setenv("LOOM_I18N_MODEL", t.DEFAULT_MODEL) + return tmp_path + + +@pytest.mark.parametrize("count", [1, 2]) +def test_present_outputs_share_one_inventory( + repo: pathlib.Path, monkeypatch: pytest.MonkeyPatch, count: int +) -> None: + for name in ("a", "b")[:count]: + output = publish(repo, f"docs/{name}.md") + sources = t.sources + discoveries = [] + + def discover() -> tuple[str, ...]: + discoveries.append(True) + return sources() + + monkeypatch.setattr(t, "sources", discover) + assert t.untranslated_pages() == [] + assert len(discoveries) == 1 + + output.write_text(output.read_text().replace("# 一", "# A").replace("# 二", "# B")) + problems = t.untranslated_pages() + assert problems and all(output.relative_to(repo).as_posix() in p for p in problems) + assert len(discoveries) == 2 + + +@pytest.mark.parametrize("replacement", ["file", "parent", "missing"]) +def test_present_pass_rechecks_source_before_reading( + repo: pathlib.Path, monkeypatch: pytest.MonkeyPatch, replacement: str +) -> None: + publish(repo, "docs/a.md") + source = repo / "docs/a.md" + outside = repo.parent / f"{repo.name}-outside" + outside.mkdir() + (outside / "a.md").write_text(source.read_text()) + sources = t.sources + + def discover() -> tuple[str, ...]: + inventory = sources() + if replacement == "parent": + source.parent.rename(repo / "original-docs") + source.parent.symlink_to(outside, target_is_directory=True) + else: + source.unlink() + if replacement == "file": + source.symlink_to(outside / "a.md") + return inventory + + read_text = pathlib.Path.read_text + reads = [] + + def read(path: pathlib.Path, *args, **kwargs): + if path == source: + reads.append(path) + return read_text(path, *args, **kwargs) + + monkeypatch.setattr(t, "sources", discover) + monkeypatch.setattr(pathlib.Path, "read_text", read) + with pytest.raises(SystemExit, match="not a regular file"): + t.untranslated_pages() + assert reads == [] + + +def test_prepare_reads_changed_source_bytes_with_the_same_membership(repo: pathlib.Path) -> None: + inventory = frozenset(t.sources()) + assert t.prepare("docs/a.md", JA, inventory=inventory)[0] == "# A\n" + before = t.blob("docs/a.md") + (repo / "docs/a.md").write_text("# Changed\n") + assert t.prepare("docs/a.md", JA, inventory=inventory)[0] == "# Changed\n" + assert t.blob("docs/a.md") != before + + +@pytest.mark.parametrize("replacement", ["file", "parent", "missing"]) +def test_rewrite_rechecks_used_english_target(repo: pathlib.Path, replacement: str) -> None: + publish(repo, "docs/b.md") + inventory = frozenset(t.sources()) + target = repo / "docs/b.md" + outside = repo.parent / f"{repo.name}-outside" + outside.mkdir() + (outside / "b.md").write_text(target.read_text()) + if replacement == "parent": + target.parent.rename(repo / "original-docs") + target.parent.symlink_to(outside, target_is_directory=True) + else: + target.unlink() + if replacement == "file": + target.symlink_to(outside / "b.md") + with pytest.raises(SystemExit, match="not a regular file"): + t.rewrite_links("[peer](b.md#b)", "docs/a.md", "ja", inventory=inventory) + + +@pytest.mark.parametrize("boundary", ["file", "parent"]) +def test_present_and_rewrite_reject_output_symlinks( + repo: pathlib.Path, monkeypatch: pytest.MonkeyPatch, boundary: str +) -> None: + output = publish(repo, "docs/b.md") + inventory = frozenset(t.sources()) + outside = repo.parent / f"{repo.name}-outside" + outside.mkdir() + (outside / "b.md").write_text(output.read_text()) + if boundary == "parent": + output.parent.rename(output.parent.with_name("original-docs")) + output.parent.symlink_to(outside, target_is_directory=True) + else: + output.unlink() + output.symlink_to(outside / "b.md") + read_text = pathlib.Path.read_text + reads = [] + + def read(path: pathlib.Path, *args, **kwargs): + if path == output: + reads.append(path) + return read_text(path, *args, **kwargs) + + monkeypatch.setattr(pathlib.Path, "read_text", read) + rewritten = t.rewrite_links("[peer](b.md#b)", "docs/a.md", "ja", inventory=inventory) + assert rewritten == f"[peer]({t.REPO_URL}/blob/main/docs/b.md#b)" + with pytest.raises(SystemExit, match="not a regular generated Markdown page"): + t.untranslated_pages() + assert reads == [] + + +@pytest.mark.parametrize("change", ["anchor", "missing", "symlink"]) +def test_rewrite_checks_live_target_between_link_occurrences( + repo: pathlib.Path, monkeypatch: pytest.MonkeyPatch, change: str +) -> None: + output = publish(repo, "docs/b.md") + inventory = frozenset(t.sources()) + source = repo / "docs/b.md" + read_text = pathlib.Path.read_text + reads = [] + + def read(path: pathlib.Path, *args, **kwargs): + text = read_text(path, *args, **kwargs) + if path == output: + reads.append(text) + if len(reads) == 1: + if change == "anchor": + output.write_text(text.replace('', "")) + else: + source.unlink() + if change == "symlink": + source.symlink_to("a.md") + return text + + monkeypatch.setattr(pathlib.Path, "read_text", read) + text = "[first](b.md#b) [second](b.md#b)" + if change == "anchor": + assert t.rewrite_links(text, "docs/a.md", "ja", inventory=inventory) == ( + f"[first](b.md#b) [second]({t.REPO_URL}/blob/main/docs/b.md#b)" + ) + assert len(reads) == 2 + else: + with pytest.raises(SystemExit, match="not a regular file"): + t.rewrite_links(text, "docs/a.md", "ja", inventory=inventory) + assert len(reads) == 1 + + +@pytest.mark.parametrize("entry", ["prepare", "rewrite"]) +def test_standalone_calls_discover_added_and_untracked_targets( + repo: pathlib.Path, monkeypatch: pytest.MonkeyPatch, entry: str +) -> None: + text = "# A\n\n[peer](c.md#c)" + (repo / "docs/a.md").write_text(text) + output = repo / t.target_path("ja", "docs/c.md") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text('') + sources = t.sources + discoveries = [] + + def discover() -> tuple[str, ...]: + inventory = sources() + discoveries.append(inventory) + return inventory + + def invoke() -> str: + if entry == "prepare": + return t.prepare("docs/a.md", JA)[0] + return t.rewrite_links(text, "docs/a.md", "ja") + + monkeypatch.setattr(t, "sources", discover) + fallback = f"[peer]({t.REPO_URL}/blob/main/docs/c.md#c)" + assert fallback in invoke() + (repo / "docs/c.md").write_text("# C\n") + git(repo, "add", "docs/c.md") + assert "[peer](c.md#c)" in invoke() + git(repo, "rm", "--cached", "docs/c.md") + assert fallback in invoke() + assert discoveries == [ + ("docs/a.md", "docs/b.md"), + ("docs/a.md", "docs/b.md", "docs/c.md"), + ("docs/a.md", "docs/b.md"), + ] + + +def test_explicit_empty_inventory_never_discovers( + repo: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + publish(repo, "docs/b.md") + + def unexpected() -> tuple[str, ...]: + pytest.fail("an explicit membership snapshot must not rediscover") + + monkeypatch.setattr(t, "sources", unexpected) + assert t.prepare("docs/a.md", JA, inventory=frozenset()) == ("# A\n", "") + assert t.rewrite_links("[peer](b.md#b)", "docs/a.md", "ja", inventory=frozenset()) == ( + f"[peer]({t.REPO_URL}/blob/main/docs/b.md#b)" + ) + + +def test_prepare_accepts_a_regular_untracked_source(repo: pathlib.Path) -> None: + (repo / "docs/draft.md").write_text("# Draft\n") + assert "docs/draft.md" not in t.sources() + assert t.prepare("docs/draft.md", JA) == ("# Draft\n", "") + assert t.prepare("docs/draft.md", JA, inventory=frozenset()) == ("# Draft\n", "") + + +def test_independent_present_passes_discover_and_validate_added_sources( + repo: pathlib.Path, monkeypatch: pytest.MonkeyPatch +) -> None: + publish(repo, "docs/a.md") + sources = t.sources + discoveries = [] + + def discover() -> tuple[str, ...]: + inventory = sources() + discoveries.append(inventory) + return inventory + + monkeypatch.setattr(t, "sources", discover) + assert t.untranslated_pages() == [] + assert discoveries == [("docs/a.md", "docs/b.md")] + (repo / "docs/c.md").write_text("# Added\n") + git(repo, "add", "docs/c.md") + output = publish(repo, "docs/c.md") + output.write_text(output.read_text().replace("# 追加", "# Added")) + discoveries.clear() + problems = t.untranslated_pages() + assert problems and all("i18n/ja/docs/c.md" in p for p in problems) + assert discoveries == [("docs/a.md", "docs/b.md", "docs/c.md")] + git(repo, "rm", "--cached", "docs/c.md") + assert t.untranslated_pages() == [] + assert discoveries[-1] == ("docs/a.md", "docs/b.md") + assert len(discoveries) == 2 From 22ad69b2b81c125914728a2fcb86a2939aacc089 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 01:13:01 +0000 Subject: [PATCH 2/4] Match stored-page fakes to the preparation signature Signed-off-by: zlareb1 --- tests/unit/test_i18n_context.py | 2 +- tests/unit/test_i18n_stored_context.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_i18n_context.py b/tests/unit/test_i18n_context.py index 985a523aa0..9ec006683e 100644 --- a/tests/unit/test_i18n_context.py +++ b/tests/unit/test_i18n_context.py @@ -406,7 +406,7 @@ def test_stored_page_check_uses_the_same_source_preservation_guard(monkeypatch, monkeypatch.setattr(t, "ROOT", tmp_path) monkeypatch.setattr(t, "sources", lambda: (source_path,)) monkeypatch.setattr(t, "LANGUAGES", (lang,)) - monkeypatch.setattr(t, "prepare", lambda source_path, lang: (source, "")) + monkeypatch.setattr(t, "prepare", lambda source_path, lang, *, inventory=None: (source, "")) output = tmp_path / t.target_path(lang.code, source_path) output.parent.mkdir(parents=True) output.write_text(source) diff --git a/tests/unit/test_i18n_stored_context.py b/tests/unit/test_i18n_stored_context.py index 0fccda771f..f337677961 100644 --- a/tests/unit/test_i18n_stored_context.py +++ b/tests/unit/test_i18n_stored_context.py @@ -51,7 +51,7 @@ def _stored_page(monkeypatch, source): read_text = Path.read_text monkeypatch.setattr(t, "sources", lambda: ("README.md",)) monkeypatch.setattr(t, "LANGUAGES", (lang,)) - monkeypatch.setattr(t, "prepare", lambda source_path, language: (source, "")) + monkeypatch.setattr(t, "prepare", lambda source_path, language, *, inventory=None: (source, "")) monkeypatch.setattr( Path, "read_text", From c3d6c479b06c02a1f4a0681d7c30f970c88500b9 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 09:58:47 +0000 Subject: [PATCH 3/4] Give each code-result lesson test its own org The two integration cases share a path and namespace. A row written by one case can enter the other case's exact result assertion when test order changes. Keep every assertion and pass a separate registered org through each case. Signed-off-by: zlareb1 --- tests/integration/test_code_result_lessons.py | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/tests/integration/test_code_result_lessons.py b/tests/integration/test_code_result_lessons.py index ead7b2d695..6e20050d66 100644 --- a/tests/integration/test_code_result_lessons.py +++ b/tests/integration/test_code_result_lessons.py @@ -14,13 +14,20 @@ pytestmark = pytest.mark.asyncio -ORG = unique_org("code-result-lessons") PATH = "server/code/app/search/handler.py" DECISION = "Decided: the search memo never stores a caller's memories, " * 6 -def _set(base: str, token: str, key: str, value: str, files: list[str], **kw: Any) -> None: - body = {"org": ORG, "namespace": "dev", "agent": "claude-code", "memory_key": key} +def _set( + org: str, + base: str, + token: str, + key: str, + value: str, + files: list[str], + **kw: Any, +) -> None: + body = {"org": org, "namespace": "dev", "agent": "claude-code", "memory_key": key} body |= {"value": value, "files": files, **kw} r = httpx.post( f"{base}/v1/memory.set", json=body, headers={"Authorization": f"Bearer {token}"}, timeout=10 @@ -34,22 +41,32 @@ async def test_lessons_match_the_result_file_by_path( from server.core.ch import get_client from server.memory.app import path_lessons + org = unique_org("code-result-lessons") await get_client().connect() - token = mint_test_token(org=ORG, namespaces=["*"], roles=["writer"]) - _set(control_plane_url, token, "notes/exact", "Pages are cut by tokens.", [PATH]) - _set(control_plane_url, token, "notes/bare", "Every handler logs its route.", ["handler.py"]) + token = mint_test_token(org=org, namespaces=["*"], roles=["writer"]) + _set(org, control_plane_url, token, "notes/exact", "Pages are cut by tokens.", [PATH]) + _set( + org, + control_plane_url, + token, + "notes/bare", + "Every handler logs its route.", + ["handler.py"], + ) _set( + org, control_plane_url, token, "notes/other", "The memory handler drops weak hits.", ["server/memory/app/loom_search/handler.py"], ) - _set(control_plane_url, token, "design/memo", DECISION, [PATH], memory_type="decision") + _set(org, control_plane_url, token, "design/memo", DECISION, [PATH], memory_type="decision") # More rows about other directories' handler.py than a small read would keep: the # exact path's rows still come back. for i in range(12): _set( + org, control_plane_url, token, f"notes/elsewhere-{i}", @@ -61,7 +78,7 @@ async def test_lessons_match_the_result_file_by_path( rows: list[dict[str, Any]] = [] for _ in range(50): rows = ( - await path_lessons.lessons_for_paths(ORG, ["dev"], "", [PATH], token_budget=2000) + await path_lessons.lessons_for_paths(org, ["dev"], "", [PATH], token_budget=2000) ).rows if len(rows) >= 3: break @@ -78,7 +95,7 @@ async def test_lessons_match_the_result_file_by_path( } assert by_key["design/memo"]["excerpt"] == " ".join(DECISION.split()) # a cap below the decision's size leaves it out, never cut - small = (await path_lessons.lessons_for_paths(ORG, ["dev"], "", [PATH], token_budget=60)).rows + small = (await path_lessons.lessons_for_paths(org, ["dev"], "", [PATH], token_budget=60)).rows assert "design/memo" not in {r["memory_key"] for r in small} @@ -93,9 +110,10 @@ async def test_a_search_keeps_a_stored_lesson_when_another_file_read_times_out( from server.memory.app import path_lessons from server.settings import settings + org = unique_org("code-result-lessons") await get_client().connect() - token = mint_test_token(org=ORG, namespaces=["*"], roles=["writer"]) - _set(control_plane_url, token, "notes/ready", "Keep the ready file's lesson.", [PATH]) + token = mint_test_token(org=org, namespaces=["*"], roles=["writer"]) + _set(org, control_plane_url, token, "notes/ready", "Keep the ready file's lesson.", [PATH]) listing_rows = path_lessons.listing_rows stopped = asyncio.Event() @@ -117,7 +135,7 @@ async def _search(inp: CodeSearchInput) -> CodeSearchOutput: wrapped = result_lessons.WithLessons(_search, result_lessons.search_paths) out = await wrapped( CodeSearchInput( - org=ORG, namespace="code-repo", query="ready file", lesson_namespaces=["dev"] + org=org, namespace="code-repo", query="ready file", lesson_namespaces=["dev"] ) ) assert any(row["memory_key"] == "notes/ready" for row in out.lessons), out.model_dump() From cde0a70be43a188cef99c97112a47d979a361458 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 10:11:19 +0000 Subject: [PATCH 4/4] Keep inventory test fixtures inside their temporary directory Signed-off-by: zlareb1 --- tests/unit/test_i18n_present_inventory.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_i18n_present_inventory.py b/tests/unit/test_i18n_present_inventory.py index 3902ff0754..9e4206dd7e 100644 --- a/tests/unit/test_i18n_present_inventory.py +++ b/tests/unit/test_i18n_present_inventory.py @@ -29,14 +29,16 @@ def publish(root: pathlib.Path, source: str) -> pathlib.Path: @pytest.fixture def repo(tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch) -> pathlib.Path: - git(tmp_path, "init", "-q", "-b", "main") - (tmp_path / "docs").mkdir() + root = tmp_path / "repo" + root.mkdir() + git(root, "init", "-q", "-b", "main") + (root / "docs").mkdir() for name in ("a", "b"): - (tmp_path / f"docs/{name}.md").write_text(f"# {name.upper()}\n") - git(tmp_path, "add", "docs") - monkeypatch.setattr(t, "ROOT", tmp_path) + (root / f"docs/{name}.md").write_text(f"# {name.upper()}\n") + git(root, "add", "docs") + monkeypatch.setattr(t, "ROOT", root) monkeypatch.setenv("LOOM_I18N_MODEL", t.DEFAULT_MODEL) - return tmp_path + return root @pytest.mark.parametrize("count", [1, 2])