diff --git a/dokimi/changelog.d/20261114_120000_antonwolfy_surrogate-digests.md b/dokimi/changelog.d/20261114_120000_antonwolfy_surrogate-digests.md new file mode 100644 index 000000000..97c47824e --- /dev/null +++ b/dokimi/changelog.d/20261114_120000_antonwolfy_surrogate-digests.md @@ -0,0 +1,10 @@ +### Fixed + +- **A remote source URL with a non-UTF-8 byte no longer stops `cache_path`** — the clone directory is hashed from the URL's UTF-8 bytes (`surrogateescape`, whatever the locale), so a surrogate-escaped URL gets its own directory instead of a `UnicodeEncodeError`; the directory of every UTF-8 URL is unchanged. +- **A harness id with a non-UTF-8 byte and a Windows-reserved character no longer stops the suite runner** — `_safe_id` hashes the id's UTF-8 bytes (`surrogateescape`), so the directory name of an id that encodes as UTF-8 is unchanged. +- **A file name or case id with a non-UTF-8 byte no longer stops the reach and skill-reach tokens** — `token` in `repos/reach.py` and `repos/skill_reach.py` hashes the UTF-8 bytes (`surrogateescape`) of its input; the token of a UTF-8 path or case id is unchanged. +- **A doc source with a non-UTF-8 byte no longer stops the URL cache key or the topics cache directory** — `URLClient._cache_key` and the question step of `EvaluationPipeline` hash the source's UTF-8 bytes (`surrogateescape`); both keys of a UTF-8 source are unchanged. +- **A matrix cell name with a lone surrogate no longer stops the plugin-delta pairing** — the pair suffix of the delta artifact in `eval/matrix_rollup.py` is hashed with `surrogatepass`, so a JSON escape such as `"n\ud800"` or `"n\udcff"` in the matrix config gets its own suffix, and the suffix of a name that encodes as UTF-8 is unchanged. +- **A symbol with a non-UTF-8 byte no longer stops `task_id` of the graph tasks** — the hash of the symbols in `repos/graph_tasks.py` uses their UTF-8 bytes (`surrogateescape`); the id of a UTF-8 symbol set is unchanged. +- **A bundle file whose name is not UTF-8 no longer stops `Bundle.measured_digest`** — the digest in `novelty/bundle.py` is taken over the UTF-8 bytes (`surrogateescape`) of the path rows; the digest of a UTF-8 path is unchanged. +- **A `\udXXX` escape in a loaded JSON or YAML file no longer stops the digests of the parsed data** — `item_digest` and `assignment_digest` (leak gate), `graph_digest` (eval coverage), `expectations_digest` (conformance), `response_cache_key` and the Harbor export's content key encode with `surrogatepass`, so a lone surrogate gets a digest of its own and every digest that encoded before is unchanged. A terminal-bench task whose `task.yaml` metadata holds such an escape still stops the Harbor export when it writes `task.toml`. diff --git a/dokimi/dokimi/caching.py b/dokimi/dokimi/caching.py index 2214113db..920d4aba1 100644 --- a/dokimi/dokimi/caching.py +++ b/dokimi/dokimi/caching.py @@ -375,7 +375,7 @@ def response_cache_key( "extra": extra or {}, } blob = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str) - return hashlib.sha256(blob.encode("utf-8")).hexdigest() + return hashlib.sha256(blob.encode("utf-8", "surrogatepass")).hexdigest() def _usage_value_fits(name: str, value: object) -> bool: diff --git a/dokimi/dokimi/eval/leakgate.py b/dokimi/dokimi/eval/leakgate.py index 8d4c012ce..b033c7f31 100644 --- a/dokimi/dokimi/eval/leakgate.py +++ b/dokimi/dokimi/eval/leakgate.py @@ -160,7 +160,7 @@ def _listed(ids: Sequence[str], limit: int = 10) -> str: def item_digest(question: Mapping[str, Any]) -> str: """SHA-256 of the question's ``grading`` block, canonically dumped: the expected answers and checks.""" body = json.dumps(question.get("grading"), ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str) - return hashlib.sha256(body.encode("utf-8")).hexdigest() + return hashlib.sha256(body.encode("utf-8", "surrogatepass")).hexdigest() def item_digests(questions: Sequence[Mapping[str, Any]]) -> dict[str, str]: @@ -173,7 +173,7 @@ def assignment_digest(assignment: Mapping[str, str], contents: Mapping[str, str] however the split file lists them, have the same digest; a changed answer or check changes it too.""" triples = [[i, assignment[i], contents[i]] for i in sorted(assignment)] body = json.dumps(triples, ensure_ascii=False, separators=(",", ":")) - return hashlib.sha256(body.encode("utf-8")).hexdigest() + return hashlib.sha256(body.encode("utf-8", "surrogatepass")).hexdigest() def _counts(assignment: Mapping[str, str]) -> dict[str, int]: diff --git a/dokimi/dokimi/eval/matrix_rollup.py b/dokimi/dokimi/eval/matrix_rollup.py index 38ff41ce5..35dddc612 100644 --- a/dokimi/dokimi/eval/matrix_rollup.py +++ b/dokimi/dokimi/eval/matrix_rollup.py @@ -87,7 +87,7 @@ def _paired_plugin_deltas( ) continue - pair_suffix = sha256(f"{baseline_name}\0{plugin_name}".encode("utf-8")).hexdigest()[:8] + pair_suffix = sha256(f"{baseline_name}\0{plugin_name}".encode("utf-8", "surrogatepass")).hexdigest()[:8] delta_path = out_dir / ( f"plugin-delta-{_safe_name(baseline_name)}-to-" f"{_safe_name(plugin_name)}-{pair_suffix}.json" diff --git a/dokimi/dokimi/eval_coverage.py b/dokimi/dokimi/eval_coverage.py index 0c91ac56b..b1ecfcf3e 100644 --- a/dokimi/dokimi/eval_coverage.py +++ b/dokimi/dokimi/eval_coverage.py @@ -69,7 +69,7 @@ def graph_digest(graph: dict) -> str: "confirmed_by": n.get("confirmed_by", "hypothesis")} for n in graph["nodes"]), key=lambda n: n["id"]) blob = json.dumps(nodes, sort_keys=True, separators=(",", ":"), ensure_ascii=False) - return hashlib.sha256(blob.encode("utf-8")).hexdigest() + return hashlib.sha256(blob.encode("utf-8", "surrogatepass")).hexdigest() _ABSENT = object() # no ``needs`` key at all: not the same as ``"needs": null``, which is malformed diff --git a/dokimi/dokimi/harnesses/harbor_export.py b/dokimi/dokimi/harnesses/harbor_export.py index de8a60735..50eddd938 100644 --- a/dokimi/dokimi/harnesses/harbor_export.py +++ b/dokimi/dokimi/harnesses/harbor_export.py @@ -97,7 +97,7 @@ def default_cache_root() -> Path: def _content_key(path: Path, resolved: str = "") -> str: """Hash of the task's files and of *resolved*, what the export writes from runtime state.""" - digest = hashlib.sha256(EXPORT_VERSION.encode() + b"\0" + resolved.encode() + b"\0") + digest = hashlib.sha256(EXPORT_VERSION.encode() + b"\0" + resolved.encode("utf-8", "surrogatepass") + b"\0") for file in sorted(p for p in path.rglob("*") if p.is_file() and not p.is_symlink()): digest.update(os.fsencode(file.relative_to(path).as_posix()) + b"\0") digest.update(hashlib.sha256(file.read_bytes()).digest()) diff --git a/dokimi/dokimi/harnesses/runner.py b/dokimi/dokimi/harnesses/runner.py index 92501893b..90fc0e56c 100644 --- a/dokimi/dokimi/harnesses/runner.py +++ b/dokimi/dokimi/harnesses/runner.py @@ -571,4 +571,4 @@ def _safe_id(value: str) -> str: name = value.replace("/", "_").replace(":", "_").replace(" ", "_") if not re.search(r'[\\<>"|?*]', name): return name - return re.sub(r'[\\<>"|?*]', "_", name) + "-" + hashlib.sha256(value.encode()).hexdigest()[:8] + return re.sub(r'[\\<>"|?*]', "_", name) + "-" + hashlib.sha256(value.encode("utf-8", "surrogateescape")).hexdigest()[:8] diff --git a/dokimi/dokimi/mcp/url_client.py b/dokimi/dokimi/mcp/url_client.py index 00f767283..297e742dc 100644 --- a/dokimi/dokimi/mcp/url_client.py +++ b/dokimi/dokimi/mcp/url_client.py @@ -132,7 +132,7 @@ def check_connection(self) -> bool: # ── Helpers ─────────────────────────────────────────────────────────── def _cache_key(self) -> str: - return hashlib.sha256(self.url.encode()).hexdigest() + return hashlib.sha256(self.url.encode("utf-8", "surrogateescape")).hexdigest() def _fetch(self) -> str: """Fetch URL (with optional caching). Returns cleaned plain text.""" diff --git a/dokimi/dokimi/novelty/bundle.py b/dokimi/dokimi/novelty/bundle.py index bd3c10125..3b6bddc55 100644 --- a/dokimi/dokimi/novelty/bundle.py +++ b/dokimi/dokimi/novelty/bundle.py @@ -247,7 +247,7 @@ def measured_digest(self) -> str: the same bytes of the same files. """ rows = "\n".join(f"{item.path}:{item.sha256}" for item in self.files) - return "sha256:" + hashlib.sha256(rows.encode("utf-8")).hexdigest() + return "sha256:" + hashlib.sha256(rows.encode("utf-8", "surrogateescape")).hexdigest() def _counts(self) -> dict: """The record's raw counts, from which every coverage figure is derived.""" diff --git a/dokimi/dokimi/orchestrator/pipeline.py b/dokimi/dokimi/orchestrator/pipeline.py index ce683de58..66b37a888 100644 --- a/dokimi/dokimi/orchestrator/pipeline.py +++ b/dokimi/dokimi/orchestrator/pipeline.py @@ -497,7 +497,7 @@ def _generate_questions(self, personas: Dict[str, Any]) -> List[Dict[str, Any]]: library_id = self._library_id(mcp_client) # The id is the registry's for every source, so the topics cache is also keyed on the source. - source_key = hashlib.sha256(self.doc_source.encode("utf-8")).hexdigest()[:16] + source_key = hashlib.sha256(self.doc_source.encode("utf-8", "surrogateescape")).hexdigest()[:16] extractor = RagasSeedExtractor(mcp_client=mcp_client, cache_dir=Path(".cache/topics") / source_key) topics = extractor.extract_topics( diff --git a/dokimi/dokimi/repos/conformance.py b/dokimi/dokimi/repos/conformance.py index 6da71dbad..88e8c452a 100644 --- a/dokimi/dokimi/repos/conformance.py +++ b/dokimi/dokimi/repos/conformance.py @@ -106,7 +106,7 @@ def load_expectations(path: Path) -> dict[str, Any]: def expectations_digest(expect: Mapping[str, Any]) -> str: """The sha256 of the expectations as canonical JSON: matrices built from different documented states differ in it.""" text = json.dumps(expect, sort_keys=True, separators=(",", ":"), ensure_ascii=False) - return hashlib.sha256(text.encode("utf-8")).hexdigest() + return hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest() def fixtures_digest(fixtures: Path) -> str: diff --git a/dokimi/dokimi/repos/graph_tasks.py b/dokimi/dokimi/repos/graph_tasks.py index c491e2b9a..0a9f33083 100644 --- a/dokimi/dokimi/repos/graph_tasks.py +++ b/dokimi/dokimi/repos/graph_tasks.py @@ -158,7 +158,7 @@ def write(p: Plan, opts: Options, code_index: dict[str, str]) -> None: def task_id(c: Candidate) -> str: """``graph___``: the hash of the symbols keeps two targets of one name (another seed, another revision) from sharing a directory.""" - digest = hashlib.sha256("\n".join(c.symbols).encode()).hexdigest()[:HASH_LEN] + digest = hashlib.sha256("\n".join(c.symbols).encode("utf-8", "surrogateescape")).hexdigest()[:HASH_LEN] slug = re.sub(r"[^a-z0-9]+", "_", c.name.lower()).strip("_") or "symbol" return f"graph_{c.kind}_{slug}"[:ID_MAX - HASH_LEN - 1] + f"_{digest}" diff --git a/dokimi/dokimi/repos/reach.py b/dokimi/dokimi/repos/reach.py index cf570098b..5a7340acf 100644 --- a/dokimi/dokimi/repos/reach.py +++ b/dokimi/dokimi/repos/reach.py @@ -198,7 +198,7 @@ def shadow_for(profile: HarnessProfile, tree: RepoTree, kinds: tuple[str, ...]) def token(salt: str, path: str, position: str) -> str: - digest = hashlib.sha256(f"{salt}\0{path}\0{position}".encode()).hexdigest() + digest = hashlib.sha256(f"{salt}\0{path}\0{position}".encode("utf-8", "surrogateescape")).hexdigest() return "RC" + digest[:6].upper() diff --git a/dokimi/dokimi/repos/skill_reach.py b/dokimi/dokimi/repos/skill_reach.py index 8d10df949..e35aedc08 100644 --- a/dokimi/dokimi/repos/skill_reach.py +++ b/dokimi/dokimi/repos/skill_reach.py @@ -233,7 +233,12 @@ def resolve_at(case: Case, dirs: dict[str, str]) -> tuple[str, str] | None: def token(salt: str, case_id: str, position: str) -> str: - digest = hashlib.sha256(f"{salt}\0{case_id}\0{position}".encode()).hexdigest() + text = f"{salt}\0{case_id}\0{position}" + try: + raw = text.encode("utf-8", "surrogateescape") + except UnicodeEncodeError: # a JSON `\ud800` escape is a surrogate `surrogateescape` does not map + raw = text.encode("utf-8", "surrogatepass") + digest = hashlib.sha256(raw).hexdigest() return ("SK" if position == "description" else "SB") + digest[:6].upper() diff --git a/dokimi/dokimi/repos/source.py b/dokimi/dokimi/repos/source.py index 16bc61bcd..78c2ca579 100644 --- a/dokimi/dokimi/repos/source.py +++ b/dokimi/dokimi/repos/source.py @@ -408,7 +408,7 @@ def default_cache_dir() -> Path: def cache_path(url: str, cache_dir: Path) -> Path: """Stable per-URL clone directory: readable slug plus a hash of the full URL.""" slug = re.sub(r"[^A-Za-z0-9._-]+", "_", url.rstrip("/").rsplit("/", 1)[-1]) or "repo" - digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12] + digest = hashlib.sha256(url.encode("utf-8", "surrogateescape")).hexdigest()[:12] return cache_dir / f"{slug.removesuffix('.git')}-{digest}.git" diff --git a/dokimi/tests/test_lone_surrogate_digests.py b/dokimi/tests/test_lone_surrogate_digests.py new file mode 100644 index 000000000..db113468d --- /dev/null +++ b/dokimi/tests/test_lone_surrogate_digests.py @@ -0,0 +1,362 @@ +# Copyright (C) 2026 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +"""A hash of text that holds a lone surrogate (a surrogate-escaped argv value or file name, a ``\\udXXX`` JSON escape) must not raise. + +The digest of every input that encodes as UTF-8 stays what it was; two different surrogates stay apart. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +posix_only = pytest.mark.skipif(sys.platform == "win32", reason="a surrogate-escaped name is a POSIX file-name form") + + +@posix_only +def test_cache_path_of_a_remote_url_with_a_non_utf8_byte_is_a_directory_name(): + from dokimi.repos.source import RepoSource, cache_path + + source = RepoSource.parse("https://example.com/n\udcff@" + "a" * 40) + assert source.location == "https://example.com/n\udcff" + assert cache_path(source.location, Path("/c")) == Path("/c/n_-4d0dae2da255.git") + assert cache_path("https://example.com/n\udcfe", Path("/c")) == Path("/c/n_-e1417c254653.git") + + +def test_cache_path_of_a_utf8_url_is_unchanged(): + from dokimi.repos.source import cache_path + + assert cache_path("https://example.com/org/repo.git", Path("/c")) == Path("/c/repo-e9497c38e4fa.git") + + +@posix_only +def test_harness_id_with_a_non_utf8_byte_and_a_reserved_character_is_a_directory_name(): + from dokimi.harnesses.runner import _safe_id + + assert _safe_id("n\udcff?") == "n\udcff_-2d5da54d" + assert _safe_id("n\udcfe?") == "n\udcfe_-f6a8699d" + + +def test_harness_id_with_a_reserved_character_keeps_its_directory_name(): + from dokimi.harnesses.runner import _safe_id + + assert _safe_id("a:b?c") == "a_b_c-f1b58422" + assert _safe_id("plain") == "plain" + + +@posix_only +def test_reach_token_of_a_file_name_with_a_non_utf8_byte_is_a_token(): + from dokimi.repos.reach import token + + assert token("s", "n\udcff", "p") == "RCE80092" + assert token("s", "n\udcfe", "p") == "RC18DC10" + + +def test_reach_token_of_a_utf8_path_is_unchanged(): + from dokimi.repos.reach import token + + assert token("s", "a/b.py", "p") == "RCC6EA29" + + +@posix_only +def test_skill_reach_token_of_a_case_id_with_a_non_utf8_byte_is_a_token(): + from dokimi.repos.skill_reach import token + + assert token("s", "n\udcff", "p") == "SBE80092" + assert token("s", "n\udcfe", "p") == "SB18DC10" + + +def test_skill_reach_token_of_a_json_escaped_high_surrogate_case_id(): + import json + + from dokimi.repos.skill_reach import token + + assert token("s", json.loads('"n\\ud800"'), "p").startswith("SB") + assert token("s", json.loads('"n\\ud800"'), "p") != token("s", json.loads('"n\\ud801"'), "p") + + +def test_skill_reach_token_of_a_utf8_case_id_is_unchanged(): + from dokimi.repos.skill_reach import token + + assert token("s", "case-1", "p") == "SB7667F2" + assert token("s", "case-1", "description") == "SKE40028" + + +@posix_only +def test_url_client_cache_key_of_a_source_with_a_non_utf8_byte_is_a_digest(): + from dokimi.mcp.url_client import URLClient + + assert URLClient("n\udcff")._cache_key() == "8a8f844082c0b6790a883eca6bd8260acd6b5ce42eca3e591fe7f9db764e6a57" + assert URLClient("n\udcfe")._cache_key() == "fad35f9379fa9598c32b2a7fa48575cf213ef99b59955fef2f454bd4a2aff8e8" + + +def test_url_client_cache_key_of_a_utf8_url_is_unchanged(): + from dokimi.mcp.url_client import URLClient + + key = URLClient("https://example.com/doc")._cache_key() + assert key == "9813a80c59ae8111adf2b881b481b0a6334db465cd7c961d98cec1830f9aa1db" # gitleaks:allow: the sha256 of a test URL + + +def _bundle_digest(path: str, sha256: str = "a") -> str: + from dokimi.novelty.bundle import Bundle, BundleFile + + files = [BundleFile(path=path, role="reference", text="x", sha256=sha256)] + return Bundle(kind="skill", name="n", description="d", frontmatter="", text="", files=files).measured_digest() + + +@posix_only +def test_bundle_measured_digest_of_a_file_name_with_a_non_utf8_byte_is_a_digest(): + assert _bundle_digest("n\udcff") == "sha256:d80bfa98e8ac7a6d87779ff266777366ec351f3af2c3c281a07505b9626f2695" + assert _bundle_digest("n\udcfe") == "sha256:30c7ad139a4dc659cf4fb1be69484b42bec251091ca030987539b9ea639d0f49" + + +def test_bundle_measured_digest_of_a_utf8_file_name_is_unchanged(): + assert _bundle_digest("references/a.md", "b" * 8) == ( + "sha256:6c2343b26584af5163cf510a83b2854ecdec51dc82b4ee23ef94f8f8c03181db") + + +def _graph_candidate(*symbols: str): + from dokimi.repos.graph_tasks import Candidate + + return Candidate("references", symbols, "run", "run", (), (), (), 1) + + +@posix_only +def test_graph_task_id_of_a_symbol_with_a_non_utf8_byte_is_an_id(): + from dokimi.repos.graph_tasks import task_id + + assert task_id(_graph_candidate("n\udcff")) == "graph_references_run_8a8f8440" + assert task_id(_graph_candidate("n\udcfe")) == "graph_references_run_fad35f93" + + +def test_graph_task_id_of_utf8_symbols_is_unchanged(): + from dokimi.repos.graph_tasks import task_id + + assert task_id(_graph_candidate("scip-python python pkg 1 mod/run().")) == "graph_references_run_89eac09d" + assert task_id(_graph_candidate("a", "b")) == "graph_references_run_7e18f737" + + +def _delta_artifact_name(baseline_name: str) -> str: + from dokimi.eval.matrix_rollup import _paired_plugin_deltas + from tests.test_matrix_rollup import _arms_artifact + + cells = [(baseline_name, Path("a"), _arms_artifact(baseline_name)), + ("p", Path("b"), _arms_artifact("p", "caveman:full"))] + deltas, warnings = _paired_plugin_deltas(cells, Path("/out")) + assert warnings == [] and len(deltas) == 1 + return Path(deltas[0]["artifact"]).name + + +def test_matrix_pair_suffix_of_a_cell_name_with_a_lone_surrogate_is_a_digest(): + assert _delta_artifact_name("n\udcff") == "plugin-delta-n-to-p-8598c7f9.json" + assert _delta_artifact_name("n\udcfe") == "plugin-delta-n-to-p-ce83522b.json" + assert _delta_artifact_name("n\ud800") == "plugin-delta-n-to-p-7e558003.json" + assert _delta_artifact_name("n\udbff") == "plugin-delta-n-to-p-e4429709.json" + + +def test_matrix_pair_suffix_of_a_utf8_cell_name_is_unchanged(): + assert _delta_artifact_name("ok") == "plugin-delta-ok-to-p-c90164a3.json" + + +# A ``\udXXX`` escape in loaded JSON or YAML is a lone surrogate, a high one (U+D800..U+DBFF) as well as a surrogate-escaped byte. +ESCAPED = json.loads('"n\\udcff"') +HIGH = json.loads('"n\\ud800"') + + +def test_leakgate_item_digest_of_a_grading_block_with_a_lone_surrogate_is_a_digest(): + from dokimi.eval.leakgate import item_digest + + assert item_digest({"grading": {"expected": ESCAPED}}) == ( + "a18bdbac1467ab19427f1e04fe443c872aa53fb437150e8b705fc0d372ce3ed8") + assert item_digest({"grading": {"expected": HIGH}}) == ( + "57c5127ebd2735361de101fa626a7c09cd127114536dc54e2fc494b81c128339") + + +def test_leakgate_item_digest_of_a_utf8_grading_block_is_unchanged(): + from dokimi.eval.leakgate import item_digest + + assert item_digest({"grading": {"expected": "é"}}) == ( + "aa7c1765ac38cd5c4aa3caf862928fb84c612929fc03bf34e8dc263a62e4c6eb") + + +def test_leakgate_assignment_digest_of_a_content_with_a_lone_surrogate_is_a_digest(): + from dokimi.eval.leakgate import assignment_digest + + assert assignment_digest({"a": "train"}, {"a": ESCAPED}) == ( + "decaed5723af65eaa5a5d7ed0c2d57c229d97bc5a9e27eafa8cb44b06206330e") + assert assignment_digest({"a": "train"}, {"a": HIGH}) == ( + "e78e6f0126b2fd72c899fcc126d132ecac713bc47210f61279d06c35b136ca4b") + assert assignment_digest({"a": "train"}, {"a": "x"}) == ( + "64594acc555505b0b1878a481ab8a02e523854c5095a04032aff909c1a6b5f8a") + + +def test_eval_graph_digest_of_a_node_text_with_a_lone_surrogate_is_a_digest(): + from dokimi.eval_coverage import graph_digest + + def graph(text: str) -> dict: + return {"nodes": [{"id": "a", "text": text, "path_class": "x"}]} + + assert graph_digest(graph(ESCAPED)) == "bdb633cf9674f5534ea741c409b6d58f8e8274b4401bd12cb696ed7bf1ebb2ab" + assert graph_digest(graph(HIGH)) == "fe60c99b86c64f7064dd623b34f9782a5cfeb377d8d75a9e0d34cc77d7601337" + assert graph_digest(graph("é")) == "8f3d895a07b207c28967c2140c969f6586f2c38a649c1cf4aa356db84ffc88ea" + + +def test_conformance_expectations_digest_of_a_value_with_a_lone_surrogate_is_a_digest(): + from dokimi.repos.conformance import expectations_digest + + assert expectations_digest({"a": ESCAPED}) == "7fe7ff64b4ed8b457fbe3e2b968dbaac3f4fdb70270df7fce0395a4f432ab833" + assert expectations_digest({"a": HIGH}) == "f7e3caab02e2e96a6cfc0ae5f5ee30e9a607fb12244b108199e78e59b87fcdb5" + assert expectations_digest({"a": "é"}) == "b3a092a6af48807fa9482b2ee140105575daa26d5b24b3c0e60a7e2dee6683b1" + + +def test_response_cache_key_of_a_message_with_a_lone_surrogate_is_a_key(): + from dokimi.caching import response_cache_key + + def key(text: str) -> str: + return response_cache_key(litellm_model="m", messages=[{"role": "user", "content": text}]) + + assert key(ESCAPED) == "3b166de6a754a5c0cd83c09b6ec0cb7bd2e30cc41a6c8214c7afa8a6427de7cc" + assert key(HIGH) == "d7e02dd5562bbfdcdafb1e9d8d81b7ce93d42dea74bee6ff47d1ef077eb37b70" + assert key("é") == "b096ff5cc3ad41f6b1f773f79e030d90735abf66a34aa8010e40b871422bdc27" + + +def test_harbor_content_key_of_a_task_toml_with_a_lone_surrogate_is_a_key(tmp_path): + from dokimi.harnesses.harbor_export import _content_key + + (tmp_path / "f.txt").write_text("x", encoding="utf-8") + assert _content_key(tmp_path, ESCAPED) == "cd926f61766a61fc" + assert _content_key(tmp_path, HIGH) == "eceb43402bdc9586" + assert _content_key(tmp_path, "é") == "0a95d7ad70b368da" + assert _content_key(tmp_path) == "11ebe38479599956" + + +def test_harbor_content_key_is_reached_by_a_task_yaml_metadata_escape(tmp_path): + from dokimi.harnesses import load_task + from dokimi.harnesses.harbor_export import _content_key, _task_toml + from tests.test_tbench_v1 import _tb1 + + task = load_task(_tb1(tmp_path / "tasks", yaml_extra='difficulty: "n\\udcff"\n')) + assert task.metadata["difficulty"] == ESCAPED + assert len(_content_key(task.path, _task_toml(task))) == 16 + + +def _topics_cache_dir(tmp_path, monkeypatch, doc_source: str) -> Path: + """The ``cache_dir`` the pipeline's question step hands the topic extractor for ``doc_source``.""" + import dokimi.questions as questions + from dokimi.orchestrator.pipeline import EvaluationPipeline + from dokimi.registry import ProductRegistry + + seen = {} + + class _StopHere(Exception): + pass + + class _Extractor: + def __init__(self, **kwargs): + seen["cache_dir"] = kwargs["cache_dir"] + + def extract_topics(self, **kwargs): + raise _StopHere + + class _Client: + requires_declared_library_id = False + + monkeypatch.setattr("dokimi.orchestrator.pipeline.create_doc_source_client", lambda source: _Client()) + monkeypatch.setattr(questions, "RagasSeedExtractor", _Extractor) + registry_path = tmp_path / "products.yaml" + registry_path.write_text("products:\n my_lib:\n name: My Lib\n description: d\n context7_id: org/lib\n", + encoding="utf-8") + pipeline = EvaluationPipeline(product="My Lib", description="d", product_key="my_lib", output_dir=tmp_path / "run", + registry=ProductRegistry(path=registry_path), doc_source=doc_source) + with pytest.raises(_StopHere): + pipeline._generate_questions({}) + return seen["cache_dir"] + + +@posix_only +def test_pipeline_topics_cache_of_a_doc_source_with_a_non_utf8_byte_gets_its_own_directory(tmp_path, monkeypatch): + first = _topics_cache_dir(tmp_path, monkeypatch, "url:https://example.com/n\udcff") + second = _topics_cache_dir(tmp_path, monkeypatch, "url:https://example.com/n\udcfe") + assert first == Path(".cache/topics/173ffc509215f580") + assert second == Path(".cache/topics/734bd48df45078ff") + + +def test_pipeline_topics_cache_of_a_utf8_doc_source_is_unchanged(tmp_path, monkeypatch): + cache_dir = _topics_cache_dir(tmp_path, monkeypatch, "url:https://example.com/doc") + assert cache_dir == Path(".cache/topics/782971a1dc6a2ae1") + + +# A POSIX process whose file-system encoding is ASCII (``LC_ALL=C PYTHONUTF8=0``): ``os.fsencode`` cannot encode "é" there. +# Every hash below is of UTF-8 bytes, so its digest must not depend on that encoding. +def _sha(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +@pytest.fixture +def ascii_fs(monkeypatch): + monkeypatch.setattr(os, "fsencode", lambda name: name.encode("ascii", "surrogateescape")) + + +def test_cache_path_of_a_non_ascii_url_does_not_depend_on_the_file_system_encoding(ascii_fs): + from dokimi.repos.source import cache_path + + url = "https://example.com/é" + assert cache_path(url, Path("/c")) == Path(f"/c/_-{_sha(url)[:12]}.git") + + +def test_harness_id_with_a_non_ascii_character_does_not_depend_on_the_file_system_encoding(ascii_fs): + from dokimi.harnesses.runner import _safe_id + + assert _safe_id("é?") == f"é_-{_sha('é?')[:8]}" + + +def test_reach_tokens_of_a_non_ascii_path_do_not_depend_on_the_file_system_encoding(ascii_fs): + from dokimi.repos.reach import token + from dokimi.repos.skill_reach import token as skill_token + + assert token("s", "é", "p") == "RC" + _sha("s\0é\0p")[:6].upper() + assert skill_token("s", "é", "p") == "SB" + _sha("s\0é\0p")[:6].upper() + + +def test_url_client_cache_key_of_a_non_ascii_url_does_not_depend_on_the_file_system_encoding(ascii_fs): + from dokimi.mcp.url_client import URLClient + + assert URLClient("https://example.com/é")._cache_key() == _sha("https://example.com/é") + + +def test_url_client_cache_key_of_a_non_ascii_url_in_an_ascii_locale_process(): + """The trigger itself: a child process with an ASCII file-system encoding.""" + code = ("import sys; sys.path.insert(0, sys.argv[1]); from dokimi.mcp.url_client import URLClient; " + "print(URLClient('https://example.com/\\u00e9')._cache_key())") + import dokimi + + env = {**os.environ, "LC_ALL": "C", "PYTHONUTF8": "0", "PYTHONCOERCECLOCALE": "0"} + done = subprocess.run([sys.executable, "-c", code, str(Path(dokimi.__file__).parent.parent)], + env=env, capture_output=True, text=True, timeout=60) + assert done.returncode == 0, done.stderr + assert done.stdout.strip() == _sha("https://example.com/é") + + +def test_bundle_measured_digest_of_a_non_ascii_file_name_does_not_depend_on_the_file_system_encoding(ascii_fs): + assert _bundle_digest("é") == "sha256:" + _sha("é:a") + + +def test_graph_task_id_of_a_non_ascii_symbol_does_not_depend_on_the_file_system_encoding(ascii_fs): + from dokimi.repos.graph_tasks import HASH_LEN, task_id + + assert task_id(_graph_candidate("é")).endswith("_" + _sha("é")[:HASH_LEN]) + + +def test_pipeline_topics_cache_of_a_non_ascii_doc_source_does_not_depend_on_the_file_system_encoding( + tmp_path, monkeypatch, ascii_fs): + cache_dir = _topics_cache_dir(tmp_path, monkeypatch, "url:https://example.com/é") + assert cache_dir == Path(".cache/topics") / _sha("url:https://example.com/é")[:16] + +