From bb292dbb88ef6bfadbf21b4a4fcd43c89b8689f6 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 07:47:20 +0000 Subject: [PATCH 1/3] Retain terminal translation origins across link changes Signed-off-by: zlareb1 --- .../architecture/documentation-translation.md | 7 ++- scripts/i18n/origins.py | 9 +-- tests/unit/test_i18n_source_origins.py | 56 +++++++++++++++++++ 3 files changed, 66 insertions(+), 6 deletions(-) diff --git a/docs/architecture/documentation-translation.md b/docs/architecture/documentation-translation.md index 1ddf40f400..b6f1ea9597 100644 --- a/docs/architecture/documentation-translation.md +++ b/docs/architecture/documentation-translation.md @@ -70,8 +70,11 @@ monotone origins. Earliest and latest embeddings of retained lines must agree at endpoint. Replacing an endpoint line or inserting text at an interior unit or separator boundary refuses; the planner does not assign that text to a neighbouring request. -A terminal append can extend the final original whole when both raw and prepared old -containers are exact proper prefixes of their current containers. The final original +A terminal append can extend the final original whole when the old raw container is an +exact proper prefix of the current raw container. Complete authenticated source maps +must locate the old raw EOF inside the current prepared container, before its EOF. +Maintained link-route changes can alter earlier prepared text or shorten the container; +each changed original whole still needs its own current-input completion. The final original must already own the last heading section and end at historical EOF. Only closed, unindented, brace-free ordinary paragraphs under that heading may be appended. Added headings, lists, tables, code blocks, HTML, reference definitions and ambiguous diff --git a/scripts/i18n/origins.py b/scripts/i18n/origins.py index cfa736a405..d10d5512cd 100644 --- a/scripts/i18n/origins.py +++ b/scripts/i18n/origins.py @@ -86,17 +86,18 @@ def _line_embeddings(old: list[str], new: list[str]) -> tuple[list[int | None], return first, list(reversed(last)) -def _terminal_append(historical, current, units, sources, positions) -> bool: +def _terminal_append(historical, current, units, sources, positions, current_positions) -> bool: from scripts.i18n import chunks before, after = sources.historical_raw, sources.current_raw + # Maintained link rewrites can change the prepared prefix without changing raw ownership. if ( not units or not before.endswith("\n") or len(after) <= len(before) or not after.startswith(before) - or not current.english.startswith(historical.english) - or len(current.english) <= len(historical.english) + or current_positions[len(before)] is None + or not current_positions[len(before)] < len(current.english) or positions[-1] != len(historical.english) or units[-1].end != len(historical.english) or not units[-1].start < units[-1].end @@ -212,7 +213,7 @@ def partition_counterparts(historical, current, units, sources: SourceOrigins): if any(p not in inverse or p in ambiguous for p in cuts): raise ValueError("original cut falls inside a source transformation") terminal_append = new[-1] == len(current.english) and _terminal_append( - historical, current, units, sources, old + historical, current, units, sources, old, new ) raw = _raw_projection( sources.historical_raw, diff --git a/tests/unit/test_i18n_source_origins.py b/tests/unit/test_i18n_source_origins.py index 9d1eef8d63..5b335635c3 100644 --- a/tests/unit/test_i18n_source_origins.py +++ b/tests/unit/test_i18n_source_origins.py @@ -413,6 +413,62 @@ def test_terminal_paragraph_append_extends_only_the_original_final_section(trail assert not plan.removed and not plan.trailing_separator +def _terminal_route_case(*, shorten=True, suffix="\nAppended.\n"): + link = "../guides/one.md" + raw = RAW.replace("Original first.", f"Original first. [Guide]({link}).").replace( + "End.", f"End. [Guide]({link})." + ) + routes = sorted(t.link_routes(link, False, PATH, "zh-CN"), key=len) + assert len(routes) == 2 + + def routed(text, native): + return t.replace_spans( + text, + [(a, b, routes[0 if native else 1]) for a, b, _, _ in t.link_targets(text)], + ) + + old, now, units, _ = _case(routed(raw + suffix, shorten), historical=routed(raw, not shorten)) + return old, now, units, SourceOrigins(raw, raw + suffix, "zh-CN") + + +@pytest.mark.parametrize("shorten", [False, True]) +def test_terminal_append_uses_authenticated_routes_when_prepared_prefix_changes(shorten): + case = _terminal_route_case(shorten=shorten) + old, now, units, _ = case + assert not now.english.startswith(old.english) + assert (len(now.english) < len(old.english)) is shorten + plan = _plan(case) + assert not plan.conflicts + assert [u.disposition for u in plan.units] == ["reuse", "changed", "reuse", "changed"] + assert tuple(u.original for u in plan.units) == units + assert plan.units[-1].scope.end == len(now.english) + assert ( + "".join( + u.separator_before + u.scope.english[u.scope.start : u.scope.end] for u in plan.units + ) + == now.english + ) + + +@pytest.mark.parametrize("problem", ["prose", "route"]) +def test_terminal_route_changes_still_require_complete_prepared_authentication(problem): + old, now, units, origins = _terminal_route_case() + text = now.english.replace( + "Original first." if problem == "prose" else "../guides/one.md", + "Forged prose." if problem == "prose" else "../guides/unrelated.md", + ) + plan = _plan((old, replace(now, english=text, sha256=_digest(text)), units, origins)) + assert plan.conflicts and not plan.units + + +@pytest.mark.parametrize( + "suffix", ["\n## Added\n\nText.\n", "\n- Item.\n", "\nText\n", "\n{: .x}\n"] +) +def test_terminal_route_changes_do_not_relax_suffix_ownership(suffix): + plan = _plan(_terminal_route_case(suffix=suffix)) + assert plan.conflicts and not plan.units + + @pytest.mark.parametrize( "suffix", [ From 3e91187f2f2d9e38ae629a79a03290d029008a4e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:42:18 +0000 Subject: [PATCH 2/3] Control the presence deadline fixture completion cutoff Wait for answered namespaces before returning the pending task at the configured cutoff. Keep the deadline verdict and elapsed limit assertions, and verify that the pending handler is cancelled. This tests deadline classification without requiring unrelated work to finish in a real scheduling window. Signed-off-by: zlareb1 --- tests/unit/test_code_presence.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_code_presence.py b/tests/unit/test_code_presence.py index 984219efd5..791d0369b7 100644 --- a/tests/unit/test_code_presence.py +++ b/tests/unit/test_code_presence.py @@ -12,6 +12,7 @@ from __future__ import annotations import asyncio +from types import SimpleNamespace from typing import Any from unittest.mock import patch @@ -377,14 +378,38 @@ def test_a_namespace_past_the_deadline_is_unknown_never_absent( ) -> None: monkeypatch.setattr(cp.settings, "code_search_fanout_timeout_ms", 50) outs = {ns: _sym_out(ns, 0) for ns in BRANCHES[:3]} - single = _Single(outs, delays={"code-clickhouse-26-2": 5.0}) + cancelled: list[str] = [] + + async def single(inp: CodeSymbolInput) -> CodeSymbolOutput: + if inp.namespace == "code-clickhouse-26-2": + try: + await asyncio.Future() + finally: + cancelled.append(inp.namespace) + return outs[inp.namespace] + + async def at_deadline(tasks: Any, **options: Any) -> tuple[set[Any], set[Any]]: + assert options == {"timeout": 0.05} + done: set[Any] = set() + pending = set(tasks) + # Cut off only after both answered namespaces finish, regardless of scheduling. + while len(done) < 2: + completed, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + done.update(completed) + return done, pending + + local_asyncio = SimpleNamespace(**vars(asyncio)) + local_asyncio.wait = at_deadline + monkeypatch.setattr(cp, "asyncio", local_asyncio) out = _run(cp.symbol_presence(_sym(presence=True, namespaces=BRANCHES[:3]), single)) by = {r["namespace"]: r for r in out.presence} assert by["code-clickhouse-26-2"]["verdict"] == "unknown" assert by["code-clickhouse-26-2"]["reason"] == "deadline" assert by["code-clickhouse-26-2"]["ms"] == 50 assert by["code-clickhouse-26-1"]["verdict"] == "absent" + assert by["code-clickhouse-26-3"]["verdict"] == "absent" assert out.summary["unknown"] == ["code-clickhouse-26-2"] + assert cancelled == ["code-clickhouse-26-2"] def test_unset_namespaces_mean_the_addressed_namespace_alone( From 71035e8a539819018adee8c12587d4dc356dad12 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:40:07 +0000 Subject: [PATCH 3/3] Replicated erasure fixtures provide isolated write leases Fenced memory inserts require a lease table in the selected schema. Give each fixture database its own Keeper root while preserving the replicated content checks. Signed-off-by: zlareb1 (cherry picked from commit 96be03c3d5fa4062c8ca44226a5e138642c86504) Signed-off-by: zlareb1 --- .../integration/test_erasure_replicated_ch.py | 5 + tests/unit/test_erasure_replicated_fixture.py | 96 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tests/unit/test_erasure_replicated_fixture.py diff --git a/tests/integration/test_erasure_replicated_ch.py b/tests/integration/test_erasure_replicated_ch.py index a1fa3f60c9..611f07969d 100644 --- a/tests/integration/test_erasure_replicated_ch.py +++ b/tests/integration/test_erasure_replicated_ch.py @@ -74,6 +74,11 @@ def db(raw: Any, monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: parameters={"db": name}, ).result_rows assert engines == [(True,)] + # Fenced memory inserts need leases isolated from the source schema's keys. + raw.command( + f"CREATE TABLE {name}._migration_lock AS {source}._migration_lock " + f"ENGINE = KeeperMap('/{name}/_migration_lock') PRIMARY KEY name" + ) monkeypatch.setattr(settings, "ch_database", name) yield name finally: diff --git a/tests/unit/test_erasure_replicated_fixture.py b/tests/unit/test_erasure_replicated_fixture.py new file mode 100644 index 0000000000..3147871591 --- /dev/null +++ b/tests/unit/test_erasure_replicated_fixture.py @@ -0,0 +1,96 @@ +"""The replicated erasure fixture supplies an isolated write-lease store.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from server.core import leases +from server.settings import settings +from tests.integration import test_erasure_replicated_ch as replicated +from tests.unit.claims_fake import FakeLockTable + + +class Schema: + def __init__(self): + self.tables = {} + self.commands = [] + + def command(self, sql): + self.commands.append(sql) + if sql.startswith("CREATE TABLE "): + self.tables[sql.split()[2]] = sql + + def query(self, sql, parameters=None): + if "engine_full" in sql: + assert parameters["db"] == "source_schema" + return SimpleNamespace(result_rows=[("ReplacingMergeTree(version) ORDER BY tuple()",)]) + assert "SELECT DISTINCT engine" in sql + flags = {"ENGINE = Replicated" in ddl for ddl in self.tables.values()} + return SimpleNamespace(result_rows=[(flag,) for flag in sorted(flags)]) + + +class FixtureLocks(FakeLockTable): + def __init__(self, schema): + super().__init__() + self.schema = schema + + async def query_one(self, sql, params=None): + if "loom._migration_lock" in sql: + if f"{settings.ch_database}._migration_lock" not in self.schema.tables: + raise RuntimeError("Unknown table _migration_lock") + return await super().query_one(sql, params) + + +@pytest.fixture +def isolated_schema(monkeypatch): + monkeypatch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, monkeypatch) + name = next(stream) + try: + yield schema, name + finally: + stream.close() + assert schema.commands[-1] == f"DROP DATABASE IF EXISTS {name} SYNC" + + +async def test_replicated_erasure_fixture_can_take_and_release_a_write_lease(isolated_schema): + schema, name = isolated_schema + keeper = FixtureLocks(schema) + lease = await leases.take("memory_insert/fixture", ttl_s=60, ch=keeper) + assert lease is not None and lease.held + assert settings.ch_database == name + await lease.release() + assert keeper.keys == {} + + +async def test_a_missing_fixture_lease_table_still_refuses(isolated_schema): + schema, name = isolated_schema + schema.tables.pop(f"{name}._migration_lock", None) + with pytest.raises(leases.LeaseUnavailable): + await leases.take("memory_insert/fixture", ttl_s=60, ch=FixtureLocks(schema)) + + +def test_fixture_lease_paths_are_separate_from_source_and_each_other(monkeypatch): + paths = [] + for _ in range(2): + with monkeypatch.context() as patch: + patch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, patch) + name = next(stream) + try: + ddl = schema.tables[f"{name}._migration_lock"] + assert "AS source_schema._migration_lock" in ddl + assert f"KeeperMap('/{name}/_migration_lock')" in ddl + assert "PRIMARY KEY name" in ddl + assert all( + "ENGINE = Replicated" in schema.tables[f"{name}.{table}"] + for table in replicated._TABLES + ) + paths.append(name) + finally: + stream.close() + assert paths[0] != paths[1]