Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions docs/architecture/documentation-translation.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,8 +79,11 @@ monotone origins. Earliest and latest embeddings of retained lines must agree at
endpoint. Replacing an endpoint line or inserting text at an interior unit or separator
boundary refuses; the planner does not assign that text to a neighbouring request.

A terminal append can extend the final original whole when both raw and prepared old
containers are exact proper prefixes of their current containers. The final original
A terminal append can extend the final original whole when the old raw container is an
exact proper prefix of the current raw container. Complete authenticated source maps
must locate the old raw EOF inside the current prepared container, before its EOF.
Maintained link-route changes can alter earlier prepared text or shorten the container;
each changed original whole still needs its own current-input completion. The final original
must already own the last heading section and end at historical EOF. Only closed,
unindented, brace-free ordinary paragraphs under that heading may be appended. Added
headings, lists, tables, code blocks, HTML, reference definitions and ambiguous
Expand Down
9 changes: 5 additions & 4 deletions scripts/i18n/origins.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,17 +86,18 @@ def _line_embeddings(old: list[str], new: list[str]) -> tuple[list[int | None],
return first, list(reversed(last))


def _terminal_append(historical, current, units, sources, positions) -> bool:
def _terminal_append(historical, current, units, sources, positions, current_positions) -> bool:
from scripts.i18n import chunks

before, after = sources.historical_raw, sources.current_raw
# Maintained link rewrites can change the prepared prefix without changing raw ownership.
if (
not units
or not before.endswith("\n")
or len(after) <= len(before)
or not after.startswith(before)
or not current.english.startswith(historical.english)
or len(current.english) <= len(historical.english)
or current_positions[len(before)] is None
or not current_positions[len(before)] < len(current.english)
or positions[-1] != len(historical.english)
or units[-1].end != len(historical.english)
or not units[-1].start < units[-1].end
Expand Down Expand Up @@ -212,7 +213,7 @@ def partition_counterparts(historical, current, units, sources: SourceOrigins):
if any(p not in inverse or p in ambiguous for p in cuts):
raise ValueError("original cut falls inside a source transformation")
terminal_append = new[-1] == len(current.english) and _terminal_append(
historical, current, units, sources, old
historical, current, units, sources, old, new
)
raw = _raw_projection(
sources.historical_raw,
Expand Down
8 changes: 8 additions & 0 deletions tests/unit/test_cell_control.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,18 @@

@pytest.mark.asyncio
async def test_uncertain_commit_closes_before_cancelling_work(monkeypatch):
from server.core import pg
from tests.unit.clock_fake import ManualClock

clock = ManualClock()
monkeypatch.setattr(cell_control, "time", clock)
monkeypatch.setattr(pg, "time", clock)

class Stalled(FakeConn):
async def execute(self, sql: str, *args: Any) -> str:
answer = await super().execute(sql, *args)
if sql == "COMMIT":
clock.advance(0.03)
try:
await asyncio.Future()
finally:
Expand Down
27 changes: 26 additions & 1 deletion tests/unit/test_code_presence.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from __future__ import annotations

import asyncio
from types import SimpleNamespace
from typing import Any
from unittest.mock import patch

Expand Down Expand Up @@ -377,14 +378,38 @@ def test_a_namespace_past_the_deadline_is_unknown_never_absent(
) -> None:
monkeypatch.setattr(cp.settings, "code_search_fanout_timeout_ms", 50)
outs = {ns: _sym_out(ns, 0) for ns in BRANCHES[:3]}
single = _Single(outs, delays={"code-clickhouse-26-2": 5.0})
cancelled: list[str] = []

async def single(inp: CodeSymbolInput) -> CodeSymbolOutput:
if inp.namespace == "code-clickhouse-26-2":
try:
await asyncio.Future()
finally:
cancelled.append(inp.namespace)
return outs[inp.namespace]

async def at_deadline(tasks: Any, **options: Any) -> tuple[set[Any], set[Any]]:
assert options == {"timeout": 0.05}
done: set[Any] = set()
pending = set(tasks)
# Cut off only after both answered namespaces finish, regardless of scheduling.
while len(done) < 2:
completed, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
done.update(completed)
return done, pending

local_asyncio = SimpleNamespace(**vars(asyncio))
local_asyncio.wait = at_deadline
monkeypatch.setattr(cp, "asyncio", local_asyncio)
out = _run(cp.symbol_presence(_sym(presence=True, namespaces=BRANCHES[:3]), single))
by = {r["namespace"]: r for r in out.presence}
assert by["code-clickhouse-26-2"]["verdict"] == "unknown"
assert by["code-clickhouse-26-2"]["reason"] == "deadline"
assert by["code-clickhouse-26-2"]["ms"] == 50
assert by["code-clickhouse-26-1"]["verdict"] == "absent"
assert by["code-clickhouse-26-3"]["verdict"] == "absent"
assert out.summary["unknown"] == ["code-clickhouse-26-2"]
assert cancelled == ["code-clickhouse-26-2"]


def test_unset_namespaces_mean_the_addressed_namespace_alone(
Expand Down
96 changes: 96 additions & 0 deletions tests/unit/test_erasure_replicated_fixture.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
"""The replicated erasure fixture supplies an isolated write-lease store."""

from __future__ import annotations

from types import SimpleNamespace

import pytest

from server.core import leases
from server.settings import settings
from tests.integration import test_erasure_replicated_ch as replicated
from tests.unit.claims_fake import FakeLockTable


class Schema:
def __init__(self):
self.tables = {}
self.commands = []

def command(self, sql):
self.commands.append(sql)
if sql.startswith("CREATE TABLE "):
self.tables[sql.split()[2]] = sql

def query(self, sql, parameters=None):
if "engine_full" in sql:
assert parameters["db"] == "source_schema"
return SimpleNamespace(result_rows=[("ReplacingMergeTree(version) ORDER BY tuple()",)])
assert "SELECT DISTINCT engine" in sql
flags = {"ENGINE = Replicated" in ddl for ddl in self.tables.values()}
return SimpleNamespace(result_rows=[(flag,) for flag in sorted(flags)])


class FixtureLocks(FakeLockTable):
def __init__(self, schema):
super().__init__()
self.schema = schema

async def query_one(self, sql, params=None):
if "loom._migration_lock" in sql:
if f"{settings.ch_database}._migration_lock" not in self.schema.tables:
raise RuntimeError("Unknown table _migration_lock")
return await super().query_one(sql, params)


@pytest.fixture
def isolated_schema(monkeypatch):
monkeypatch.setattr(settings, "ch_database", "source_schema")
schema = Schema()
stream = replicated.db.__wrapped__(schema, monkeypatch)
name = next(stream)
try:
yield schema, name
finally:
stream.close()
assert schema.commands[-1] == f"DROP DATABASE IF EXISTS {name} SYNC"


async def test_replicated_erasure_fixture_can_take_and_release_a_write_lease(isolated_schema):
schema, name = isolated_schema
keeper = FixtureLocks(schema)
lease = await leases.take("memory_insert/fixture", ttl_s=60, ch=keeper)
assert lease is not None and lease.held
assert settings.ch_database == name
await lease.release()
assert keeper.keys == {}


async def test_a_missing_fixture_lease_table_still_refuses(isolated_schema):
schema, name = isolated_schema
schema.tables.pop(f"{name}._migration_lock", None)
with pytest.raises(leases.LeaseUnavailable):
await leases.take("memory_insert/fixture", ttl_s=60, ch=FixtureLocks(schema))


def test_fixture_lease_paths_are_separate_from_source_and_each_other(monkeypatch):
paths = []
for _ in range(2):
with monkeypatch.context() as patch:
patch.setattr(settings, "ch_database", "source_schema")
schema = Schema()
stream = replicated.db.__wrapped__(schema, patch)
name = next(stream)
try:
ddl = schema.tables[f"{name}._migration_lock"]
assert "AS source_schema._migration_lock" in ddl
assert f"KeeperMap('/loom/tests/{name}/migration_lock')" in ddl
assert "PRIMARY KEY name" in ddl
assert all(
"ENGINE = Replicated" in schema.tables[f"{name}.{table}"]
for table in replicated._TABLES
)
paths.append(name)
finally:
stream.close()
assert paths[0] != paths[1]
56 changes: 56 additions & 0 deletions tests/unit/test_i18n_source_origins.py
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,62 @@ def test_terminal_paragraph_append_extends_only_the_original_final_section(trail
assert not plan.removed and not plan.trailing_separator


def _terminal_route_case(*, shorten=True, suffix="\nAppended.\n"):
link = "../guides/one.md"
raw = RAW.replace("Original first.", f"Original first. [Guide]({link}).").replace(
"End.", f"End. [Guide]({link})."
)
routes = sorted(t.link_routes(link, False, PATH, "zh-CN"), key=len)
assert len(routes) == 2

def routed(text, native):
return t.replace_spans(
text,
[(a, b, routes[0 if native else 1]) for a, b, _, _ in t.link_targets(text)],
)

old, now, units, _ = _case(routed(raw + suffix, shorten), historical=routed(raw, not shorten))
return old, now, units, SourceOrigins(raw, raw + suffix, "zh-CN")


@pytest.mark.parametrize("shorten", [False, True])
def test_terminal_append_uses_authenticated_routes_when_prepared_prefix_changes(shorten):
case = _terminal_route_case(shorten=shorten)
old, now, units, _ = case
assert not now.english.startswith(old.english)
assert (len(now.english) < len(old.english)) is shorten
plan = _plan(case)
assert not plan.conflicts
assert [u.disposition for u in plan.units] == ["reuse", "changed", "reuse", "changed"]
assert tuple(u.original for u in plan.units) == units
assert plan.units[-1].scope.end == len(now.english)
assert (
"".join(
u.separator_before + u.scope.english[u.scope.start : u.scope.end] for u in plan.units
)
== now.english
)


@pytest.mark.parametrize("problem", ["prose", "route"])
def test_terminal_route_changes_still_require_complete_prepared_authentication(problem):
old, now, units, origins = _terminal_route_case()
text = now.english.replace(
"Original first." if problem == "prose" else "../guides/one.md",
"Forged prose." if problem == "prose" else "../guides/unrelated.md",
)
plan = _plan((old, replace(now, english=text, sha256=_digest(text)), units, origins))
assert plan.conflicts and not plan.units


@pytest.mark.parametrize(
"suffix", ["\n## Added\n\nText.\n", "\n- Item.\n", "\n<b>Text</b>\n", "\n{: .x}\n"]
)
def test_terminal_route_changes_do_not_relax_suffix_ownership(suffix):
plan = _plan(_terminal_route_case(suffix=suffix))
assert plan.conflicts and not plan.units


@pytest.mark.parametrize(
"suffix",
[
Expand Down