diff --git a/docs/architecture/documentation-translation.md b/docs/architecture/documentation-translation.md index fd9ad74726..f8fbf16351 100644 --- a/docs/architecture/documentation-translation.md +++ b/docs/architecture/documentation-translation.md @@ -79,8 +79,11 @@ monotone origins. Earliest and latest embeddings of retained lines must agree at endpoint. Replacing an endpoint line or inserting text at an interior unit or separator boundary refuses; the planner does not assign that text to a neighbouring request. -A terminal append can extend the final original whole when both raw and prepared old -containers are exact proper prefixes of their current containers. The final original +A terminal append can extend the final original whole when the old raw container is an +exact proper prefix of the current raw container. Complete authenticated source maps +must locate the old raw EOF inside the current prepared container, before its EOF. +Maintained link-route changes can alter earlier prepared text or shorten the container; +each changed original whole still needs its own current-input completion. The final original must already own the last heading section and end at historical EOF. Only closed, unindented, brace-free ordinary paragraphs under that heading may be appended. Added headings, lists, tables, code blocks, HTML, reference definitions and ambiguous diff --git a/scripts/i18n/origins.py b/scripts/i18n/origins.py index cfa736a405..d10d5512cd 100644 --- a/scripts/i18n/origins.py +++ b/scripts/i18n/origins.py @@ -86,17 +86,18 @@ def _line_embeddings(old: list[str], new: list[str]) -> tuple[list[int | None], return first, list(reversed(last)) -def _terminal_append(historical, current, units, sources, positions) -> bool: +def _terminal_append(historical, current, units, sources, positions, current_positions) -> bool: from scripts.i18n import chunks before, after = sources.historical_raw, sources.current_raw + # Maintained link rewrites can change the prepared prefix without changing raw ownership. if ( not units or not before.endswith("\n") or len(after) <= len(before) or not after.startswith(before) - or not current.english.startswith(historical.english) - or len(current.english) <= len(historical.english) + or current_positions[len(before)] is None + or not current_positions[len(before)] < len(current.english) or positions[-1] != len(historical.english) or units[-1].end != len(historical.english) or not units[-1].start < units[-1].end @@ -212,7 +213,7 @@ def partition_counterparts(historical, current, units, sources: SourceOrigins): if any(p not in inverse or p in ambiguous for p in cuts): raise ValueError("original cut falls inside a source transformation") terminal_append = new[-1] == len(current.english) and _terminal_append( - historical, current, units, sources, old + historical, current, units, sources, old, new ) raw = _raw_projection( sources.historical_raw, diff --git a/tests/unit/test_cell_control.py b/tests/unit/test_cell_control.py index 8e0c5fd4e7..e59cea5b1f 100644 --- a/tests/unit/test_cell_control.py +++ b/tests/unit/test_cell_control.py @@ -15,10 +15,18 @@ @pytest.mark.asyncio async def test_uncertain_commit_closes_before_cancelling_work(monkeypatch): + from server.core import pg + from tests.unit.clock_fake import ManualClock + + clock = ManualClock() + monkeypatch.setattr(cell_control, "time", clock) + monkeypatch.setattr(pg, "time", clock) + class Stalled(FakeConn): async def execute(self, sql: str, *args: Any) -> str: answer = await super().execute(sql, *args) if sql == "COMMIT": + clock.advance(0.03) try: await asyncio.Future() finally: diff --git a/tests/unit/test_code_presence.py b/tests/unit/test_code_presence.py index 984219efd5..791d0369b7 100644 --- a/tests/unit/test_code_presence.py +++ b/tests/unit/test_code_presence.py @@ -12,6 +12,7 @@ from __future__ import annotations import asyncio +from types import SimpleNamespace from typing import Any from unittest.mock import patch @@ -377,14 +378,38 @@ def test_a_namespace_past_the_deadline_is_unknown_never_absent( ) -> None: monkeypatch.setattr(cp.settings, "code_search_fanout_timeout_ms", 50) outs = {ns: _sym_out(ns, 0) for ns in BRANCHES[:3]} - single = _Single(outs, delays={"code-clickhouse-26-2": 5.0}) + cancelled: list[str] = [] + + async def single(inp: CodeSymbolInput) -> CodeSymbolOutput: + if inp.namespace == "code-clickhouse-26-2": + try: + await asyncio.Future() + finally: + cancelled.append(inp.namespace) + return outs[inp.namespace] + + async def at_deadline(tasks: Any, **options: Any) -> tuple[set[Any], set[Any]]: + assert options == {"timeout": 0.05} + done: set[Any] = set() + pending = set(tasks) + # Cut off only after both answered namespaces finish, regardless of scheduling. + while len(done) < 2: + completed, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + done.update(completed) + return done, pending + + local_asyncio = SimpleNamespace(**vars(asyncio)) + local_asyncio.wait = at_deadline + monkeypatch.setattr(cp, "asyncio", local_asyncio) out = _run(cp.symbol_presence(_sym(presence=True, namespaces=BRANCHES[:3]), single)) by = {r["namespace"]: r for r in out.presence} assert by["code-clickhouse-26-2"]["verdict"] == "unknown" assert by["code-clickhouse-26-2"]["reason"] == "deadline" assert by["code-clickhouse-26-2"]["ms"] == 50 assert by["code-clickhouse-26-1"]["verdict"] == "absent" + assert by["code-clickhouse-26-3"]["verdict"] == "absent" assert out.summary["unknown"] == ["code-clickhouse-26-2"] + assert cancelled == ["code-clickhouse-26-2"] def test_unset_namespaces_mean_the_addressed_namespace_alone( diff --git a/tests/unit/test_erasure_replicated_fixture.py b/tests/unit/test_erasure_replicated_fixture.py new file mode 100644 index 0000000000..02c93ce77a --- /dev/null +++ b/tests/unit/test_erasure_replicated_fixture.py @@ -0,0 +1,96 @@ +"""The replicated erasure fixture supplies an isolated write-lease store.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from server.core import leases +from server.settings import settings +from tests.integration import test_erasure_replicated_ch as replicated +from tests.unit.claims_fake import FakeLockTable + + +class Schema: + def __init__(self): + self.tables = {} + self.commands = [] + + def command(self, sql): + self.commands.append(sql) + if sql.startswith("CREATE TABLE "): + self.tables[sql.split()[2]] = sql + + def query(self, sql, parameters=None): + if "engine_full" in sql: + assert parameters["db"] == "source_schema" + return SimpleNamespace(result_rows=[("ReplacingMergeTree(version) ORDER BY tuple()",)]) + assert "SELECT DISTINCT engine" in sql + flags = {"ENGINE = Replicated" in ddl for ddl in self.tables.values()} + return SimpleNamespace(result_rows=[(flag,) for flag in sorted(flags)]) + + +class FixtureLocks(FakeLockTable): + def __init__(self, schema): + super().__init__() + self.schema = schema + + async def query_one(self, sql, params=None): + if "loom._migration_lock" in sql: + if f"{settings.ch_database}._migration_lock" not in self.schema.tables: + raise RuntimeError("Unknown table _migration_lock") + return await super().query_one(sql, params) + + +@pytest.fixture +def isolated_schema(monkeypatch): + monkeypatch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, monkeypatch) + name = next(stream) + try: + yield schema, name + finally: + stream.close() + assert schema.commands[-1] == f"DROP DATABASE IF EXISTS {name} SYNC" + + +async def test_replicated_erasure_fixture_can_take_and_release_a_write_lease(isolated_schema): + schema, name = isolated_schema + keeper = FixtureLocks(schema) + lease = await leases.take("memory_insert/fixture", ttl_s=60, ch=keeper) + assert lease is not None and lease.held + assert settings.ch_database == name + await lease.release() + assert keeper.keys == {} + + +async def test_a_missing_fixture_lease_table_still_refuses(isolated_schema): + schema, name = isolated_schema + schema.tables.pop(f"{name}._migration_lock", None) + with pytest.raises(leases.LeaseUnavailable): + await leases.take("memory_insert/fixture", ttl_s=60, ch=FixtureLocks(schema)) + + +def test_fixture_lease_paths_are_separate_from_source_and_each_other(monkeypatch): + paths = [] + for _ in range(2): + with monkeypatch.context() as patch: + patch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, patch) + name = next(stream) + try: + ddl = schema.tables[f"{name}._migration_lock"] + assert "AS source_schema._migration_lock" in ddl + assert f"KeeperMap('/loom/tests/{name}/migration_lock')" in ddl + assert "PRIMARY KEY name" in ddl + assert all( + "ENGINE = Replicated" in schema.tables[f"{name}.{table}"] + for table in replicated._TABLES + ) + paths.append(name) + finally: + stream.close() + assert paths[0] != paths[1] diff --git a/tests/unit/test_i18n_source_origins.py b/tests/unit/test_i18n_source_origins.py index 9d1eef8d63..5b335635c3 100644 --- a/tests/unit/test_i18n_source_origins.py +++ b/tests/unit/test_i18n_source_origins.py @@ -413,6 +413,62 @@ def test_terminal_paragraph_append_extends_only_the_original_final_section(trail assert not plan.removed and not plan.trailing_separator +def _terminal_route_case(*, shorten=True, suffix="\nAppended.\n"): + link = "../guides/one.md" + raw = RAW.replace("Original first.", f"Original first. [Guide]({link}).").replace( + "End.", f"End. [Guide]({link})." + ) + routes = sorted(t.link_routes(link, False, PATH, "zh-CN"), key=len) + assert len(routes) == 2 + + def routed(text, native): + return t.replace_spans( + text, + [(a, b, routes[0 if native else 1]) for a, b, _, _ in t.link_targets(text)], + ) + + old, now, units, _ = _case(routed(raw + suffix, shorten), historical=routed(raw, not shorten)) + return old, now, units, SourceOrigins(raw, raw + suffix, "zh-CN") + + +@pytest.mark.parametrize("shorten", [False, True]) +def test_terminal_append_uses_authenticated_routes_when_prepared_prefix_changes(shorten): + case = _terminal_route_case(shorten=shorten) + old, now, units, _ = case + assert not now.english.startswith(old.english) + assert (len(now.english) < len(old.english)) is shorten + plan = _plan(case) + assert not plan.conflicts + assert [u.disposition for u in plan.units] == ["reuse", "changed", "reuse", "changed"] + assert tuple(u.original for u in plan.units) == units + assert plan.units[-1].scope.end == len(now.english) + assert ( + "".join( + u.separator_before + u.scope.english[u.scope.start : u.scope.end] for u in plan.units + ) + == now.english + ) + + +@pytest.mark.parametrize("problem", ["prose", "route"]) +def test_terminal_route_changes_still_require_complete_prepared_authentication(problem): + old, now, units, origins = _terminal_route_case() + text = now.english.replace( + "Original first." if problem == "prose" else "../guides/one.md", + "Forged prose." if problem == "prose" else "../guides/unrelated.md", + ) + plan = _plan((old, replace(now, english=text, sha256=_digest(text)), units, origins)) + assert plan.conflicts and not plan.units + + +@pytest.mark.parametrize( + "suffix", ["\n## Added\n\nText.\n", "\n- Item.\n", "\nText\n", "\n{: .x}\n"] +) +def test_terminal_route_changes_do_not_relax_suffix_ownership(suffix): + plan = _plan(_terminal_route_case(suffix=suffix)) + assert plan.conflicts and not plan.units + + @pytest.mark.parametrize( "suffix", [