fix(digest): hashes of text with a lone surrogate return a digest instead of raising UnicodeEncodeError

dokimi/changelog.d/20261114_120000_antonwolfy_surrogate-digests.md

@@ -0,0 +1,10 @@
+### Fixed
+
+- **A remote source URL with a non-UTF-8 byte no longer stops `cache_path`** — the clone directory is hashed from the URL's UTF-8 bytes (`surrogateescape`, whatever the locale), so a surrogate-escaped URL gets its own directory instead of a `UnicodeEncodeError`; the directory of every UTF-8 URL is unchanged.
+- **A harness id with a non-UTF-8 byte and a Windows-reserved character no longer stops the suite runner** — `_safe_id` hashes the id's UTF-8 bytes (`surrogateescape`), so the directory name of an id that encodes as UTF-8 is unchanged.
+- **A file name or case id with a non-UTF-8 byte no longer stops the reach and skill-reach tokens** — `token` in `repos/reach.py` and `repos/skill_reach.py` hashes the UTF-8 bytes (`surrogateescape`) of its input; the token of a UTF-8 path or case id is unchanged.
+- **A doc source with a non-UTF-8 byte no longer stops the URL cache key or the topics cache directory** — `URLClient._cache_key` and the question step of `EvaluationPipeline` hash the source's UTF-8 bytes (`surrogateescape`); both keys of a UTF-8 source are unchanged.
+- **A matrix cell name with a lone surrogate no longer stops the plugin-delta pairing** — the pair suffix of the delta artifact in `eval/matrix_rollup.py` is hashed with `surrogatepass`, so a JSON escape such as `"n\ud800"` or `"n\udcff"` in the matrix config gets its own suffix, and the suffix of a name that encodes as UTF-8 is unchanged.
+- **A symbol with a non-UTF-8 byte no longer stops `task_id` of the graph tasks** — the hash of the symbols in `repos/graph_tasks.py` uses their UTF-8 bytes (`surrogateescape`); the id of a UTF-8 symbol set is unchanged.
+- **A bundle file whose name is not UTF-8 no longer stops `Bundle.measured_digest`** — the digest in `novelty/bundle.py` is taken over the UTF-8 bytes (`surrogateescape`) of the path rows; the digest of a UTF-8 path is unchanged.
+- **A `\udXXX` escape in a loaded JSON or YAML file no longer stops the digests of the parsed data** — `item_digest` and `assignment_digest` (leak gate), `graph_digest` (eval coverage), `expectations_digest` (conformance), `response_cache_key` and the Harbor export's content key encode with `surrogatepass`, so a lone surrogate gets a digest of its own and every digest that encoded before is unchanged. A terminal-bench task whose `task.yaml` metadata holds such an escape still stops the Harbor export when it writes `task.toml`.

dokimi/dokimi/caching.py

@@ -375,7 +375,7 @@ def response_cache_key(
         "extra": extra or {},
     }
     blob = json.dumps(payload, sort_keys=True, ensure_ascii=False, default=str)
-    return hashlib.sha256(blob.encode("utf-8")).hexdigest()
+    return hashlib.sha256(blob.encode("utf-8", "surrogatepass")).hexdigest()
 
 
 def _usage_value_fits(name: str, value: object) -> bool:

dokimi/dokimi/eval/leakgate.py

@@ -160,7 +160,7 @@ def _listed(ids: Sequence[str], limit: int = 10) -> str:
 def item_digest(question: Mapping[str, Any]) -> str:
     """SHA-256 of the question's ``grading`` block, canonically dumped: the expected answers and checks."""
     body = json.dumps(question.get("grading"), ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
-    return hashlib.sha256(body.encode("utf-8")).hexdigest()
+    return hashlib.sha256(body.encode("utf-8", "surrogatepass")).hexdigest()
 
 
 def item_digests(questions: Sequence[Mapping[str, Any]]) -> dict[str, str]:
@@ -173,7 +173,7 @@ def assignment_digest(assignment: Mapping[str, str], contents: Mapping[str, str]
     however the split file lists them, have the same digest; a changed answer or check changes it too."""
     triples = [[i, assignment[i], contents[i]] for i in sorted(assignment)]
     body = json.dumps(triples, ensure_ascii=False, separators=(",", ":"))
-    return hashlib.sha256(body.encode("utf-8")).hexdigest()
+    return hashlib.sha256(body.encode("utf-8", "surrogatepass")).hexdigest()
 
 
 def _counts(assignment: Mapping[str, str]) -> dict[str, int]:

dokimi/dokimi/eval/matrix_rollup.py

@@ -87,7 +87,7 @@ def _paired_plugin_deltas(
             )
             continue
 
-        pair_suffix = sha256(f"{baseline_name}\0{plugin_name}".encode("utf-8")).hexdigest()[:8]
+        pair_suffix = sha256(f"{baseline_name}\0{plugin_name}".encode("utf-8", "surrogatepass")).hexdigest()[:8]
         delta_path = out_dir / (
             f"plugin-delta-{_safe_name(baseline_name)}-to-"
             f"{_safe_name(plugin_name)}-{pair_suffix}.json"

dokimi/dokimi/eval_coverage.py

@@ -69,7 +69,7 @@ def graph_digest(graph: dict) -> str:
                      "confirmed_by": n.get("confirmed_by", "hypothesis")}
                     for n in graph["nodes"]), key=lambda n: n["id"])
     blob = json.dumps(nodes, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
-    return hashlib.sha256(blob.encode("utf-8")).hexdigest()
+    return hashlib.sha256(blob.encode("utf-8", "surrogatepass")).hexdigest()
 
 
 _ABSENT = object()  # no ``needs`` key at all: not the same as ``"needs": null``, which is malformed

dokimi/dokimi/harnesses/harbor_export.py

@@ -97,7 +97,7 @@ def default_cache_root() -> Path:
 
 def _content_key(path: Path, resolved: str = "") -> str:
     """Hash of the task's files and of *resolved*, what the export writes from runtime state."""
-    digest = hashlib.sha256(EXPORT_VERSION.encode() + b"\0" + resolved.encode() + b"\0")
+    digest = hashlib.sha256(EXPORT_VERSION.encode() + b"\0" + resolved.encode("utf-8", "surrogatepass") + b"\0")
     for file in sorted(p for p in path.rglob("*") if p.is_file() and not p.is_symlink()):
         digest.update(os.fsencode(file.relative_to(path).as_posix()) + b"\0")
         digest.update(hashlib.sha256(file.read_bytes()).digest())

dokimi/dokimi/harnesses/runner.py

@@ -571,4 +571,4 @@ def _safe_id(value: str) -> str:
     name = value.replace("/", "_").replace(":", "_").replace(" ", "_")
     if not re.search(r'[\\<>"|?*]', name):
         return name
-    return re.sub(r'[\\<>"|?*]', "_", name) + "-" + hashlib.sha256(value.encode()).hexdigest()[:8]
+    return re.sub(r'[\\<>"|?*]', "_", name) + "-" + hashlib.sha256(value.encode("utf-8", "surrogateescape")).hexdigest()[:8]

dokimi/dokimi/mcp/url_client.py

@@ -132,7 +132,7 @@ def check_connection(self) -> bool:
     # ── Helpers ───────────────────────────────────────────────────────────
 
     def _cache_key(self) -> str:
-        return hashlib.sha256(self.url.encode()).hexdigest()
+        return hashlib.sha256(self.url.encode("utf-8", "surrogateescape")).hexdigest()
 
     def _fetch(self) -> str:
         """Fetch URL (with optional caching). Returns cleaned plain text."""

dokimi/dokimi/novelty/bundle.py

@@ -247,7 +247,7 @@ def measured_digest(self) -> str:
         the same bytes of the same files.
         """
         rows = "\n".join(f"{item.path}:{item.sha256}" for item in self.files)
-        return "sha256:" + hashlib.sha256(rows.encode("utf-8")).hexdigest()
+        return "sha256:" + hashlib.sha256(rows.encode("utf-8", "surrogateescape")).hexdigest()
 
     def _counts(self) -> dict:
         """The record's raw counts, from which every coverage figure is derived."""

dokimi/dokimi/orchestrator/pipeline.py

@@ -497,7 +497,7 @@ def _generate_questions(self, personas: Dict[str, Any]) -> List[Dict[str, Any]]:
         library_id = self._library_id(mcp_client)
 
         # The id is the registry's for every source, so the topics cache is also keyed on the source.
-        source_key = hashlib.sha256(self.doc_source.encode("utf-8")).hexdigest()[:16]
+        source_key = hashlib.sha256(self.doc_source.encode("utf-8", "surrogateescape")).hexdigest()[:16]
         extractor = RagasSeedExtractor(mcp_client=mcp_client,
                                        cache_dir=Path(".cache/topics") / source_key)
         topics = extractor.extract_topics(

dokimi/dokimi/repos/conformance.py

@@ -106,7 +106,7 @@ def load_expectations(path: Path) -> dict[str, Any]:
 def expectations_digest(expect: Mapping[str, Any]) -> str:
     """The sha256 of the expectations as canonical JSON: matrices built from different documented states differ in it."""
     text = json.dumps(expect, sort_keys=True, separators=(",", ":"), ensure_ascii=False)
-    return hashlib.sha256(text.encode("utf-8")).hexdigest()
+    return hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest()
 
 
 def fixtures_digest(fixtures: Path) -> str:

dokimi/dokimi/repos/graph_tasks.py

@@ -158,7 +158,7 @@ def write(p: Plan, opts: Options, code_index: dict[str, str]) -> None:
 def task_id(c: Candidate) -> str:
     """``graph_<kind>_<name>_<hash>``: the hash of the symbols keeps two targets of one name (another seed,
     another revision) from sharing a directory."""
-    digest = hashlib.sha256("\n".join(c.symbols).encode()).hexdigest()[:HASH_LEN]
+    digest = hashlib.sha256("\n".join(c.symbols).encode("utf-8", "surrogateescape")).hexdigest()[:HASH_LEN]
     slug = re.sub(r"[^a-z0-9]+", "_", c.name.lower()).strip("_") or "symbol"
     return f"graph_{c.kind}_{slug}"[:ID_MAX - HASH_LEN - 1] + f"_{digest}"
 

dokimi/dokimi/repos/reach.py

@@ -198,7 +198,7 @@ def shadow_for(profile: HarnessProfile, tree: RepoTree, kinds: tuple[str, ...])
 
 
 def token(salt: str, path: str, position: str) -> str:
-    digest = hashlib.sha256(f"{salt}\0{path}\0{position}".encode()).hexdigest()
+    digest = hashlib.sha256(f"{salt}\0{path}\0{position}".encode("utf-8", "surrogateescape")).hexdigest()
     return "RC" + digest[:6].upper()
 
 

dokimi/dokimi/repos/skill_reach.py

@@ -233,7 +233,12 @@ def resolve_at(case: Case, dirs: dict[str, str]) -> tuple[str, str] | None:
 
 
 def token(salt: str, case_id: str, position: str) -> str:
-    digest = hashlib.sha256(f"{salt}\0{case_id}\0{position}".encode()).hexdigest()
+    text = f"{salt}\0{case_id}\0{position}"
+    try:
+        raw = text.encode("utf-8", "surrogateescape")
+    except UnicodeEncodeError:  # a JSON `\ud800` escape is a surrogate `surrogateescape` does not map
+        raw = text.encode("utf-8", "surrogatepass")
+    digest = hashlib.sha256(raw).hexdigest()
     return ("SK" if position == "description" else "SB") + digest[:6].upper()
 
 

dokimi/dokimi/repos/source.py

@@ -408,7 +408,7 @@ def default_cache_dir() -> Path:
 def cache_path(url: str, cache_dir: Path) -> Path:
     """Stable per-URL clone directory: readable slug plus a hash of the full URL."""
     slug = re.sub(r"[^A-Za-z0-9._-]+", "_", url.rstrip("/").rsplit("/", 1)[-1]) or "repo"
-    digest = hashlib.sha256(url.encode("utf-8")).hexdigest()[:12]
+    digest = hashlib.sha256(url.encode("utf-8", "surrogateescape")).hexdigest()[:12]
     return cache_dir / f"{slug.removesuffix('.git')}-{digest}.git"