From 1611c10388973b411eb197f1a59d06d2a02d43d7 Mon Sep 17 00:00:00 2001
From: Panos Vagenas <pva@zurich.ibm.com>
Date: Thu, 13 Aug 2026 11:32:52 +0200
Subject: [PATCH] expose DocLang special-token vocabulary for tokenizer
 integration

- Add doclang.tokenization.get_special_tokens(), a spec.md-derived list of
  special tokens ready to plug into any tokenizer's added-tokens API
- Drop <smiles>/</smiles> from spec.md's Token vocabulary table: it
  contradicted the custom-vocabulary namespacing guidance stated a few
  paragraphs earlier, and a properly namespaced example already exists
  elsewhere in the spec
- Test the vocabulary against spec.md's own table so the two can't drift

Signed-off-by: Panos Vagenas <pva@zurich.ibm.com>
---
 doclang/__init__.py        |   2 +
 doclang/tokenization.py    | 128 +++++++++++++++++++++++++++++++++++++
 spec.md                    |   2 -
 tests/test_tokenization.py |  46 +++++++++++++
 4 files changed, 176 insertions(+), 2 deletions(-)
 create mode 100644 doclang/tokenization.py
 create mode 100644 tests/test_tokenization.py

diff --git a/doclang/__init__.py b/doclang/__init__.py
index 611bd2f..edcc478 100644
--- a/doclang/__init__.py
+++ b/doclang/__init__.py
@@ -2,6 +2,7 @@
 
 from doclang.packaging import PackagingError, pack
 from doclang.schematron import SchematronBackendNotFound, SchematronValidator, SchematronViolation
+from doclang.tokenization import get_special_tokens
 from doclang.validation import ValidationError, validate
 
 __all__ = [
@@ -10,6 +11,7 @@
     "SchematronValidator",
     "SchematronViolation",
     "ValidationError",
+    "get_special_tokens",
     "pack",
     "validate",
 ]
diff --git a/doclang/tokenization.py b/doclang/tokenization.py
new file mode 100644
index 0000000..81d603a
--- /dev/null
+++ b/doclang/tokenization.py
@@ -0,0 +1,128 @@
+"""DocLang special-token vocabulary, per spec.md's (non-normative) "Token vocabulary" recommendation."""
+
+from __future__ import annotations
+
+DEFAULT_MAX_RESOLUTION = 512
+
+_HEADING_LEVELS = range(2, 7)
+_FIELD_HEADING_LEVELS = range(2, 7)
+
+_FIXED_TOKENS: tuple[str, ...] = (
+    "<doclang>",
+    "</doclang>",
+    "<page_break/>",
+    '"/>',
+    "<text>",
+    "</text>",
+    "<heading>",
+    *(f'<heading level="{level}">' for level in _HEADING_LEVELS),
+    "</heading>",
+    "<footnote>",
+    "</footnote>",
+    "<page_header>",
+    "</page_header>",
+    "<page_footer>",
+    "</page_footer>",
+    "<field_region>",
+    "</field_region>",
+    "<list>",
+    '<list class="ordered">',
+    "</list>",
+    "<table>",
+    "</table>",
+    "<index>",
+    "</index>",
+    "<formula>",
+    "</formula>",
+    '<code><label value="',
+    "</code>",
+    '<picture><label value="',
+    '<picture class="chart"><label value="',
+    "</picture>",
+    "<marker>",
+    "</marker>",
+    "<group>",
+    "</group>",
+    "<field_heading>",
+    *(f'<field_heading level="{level}">' for level in _FIELD_HEADING_LEVELS),
+    "</field_heading>",
+    "<field_item>",
+    "</field_item>",
+    "<key>",
+    "</key>",
+    "<value>",
+    '<value class="fillable">',
+    "</value>",
+    "<hint>",
+    "</hint>",
+    "<caption>",
+    "</caption>",
+    "<description>",
+    "</description>",
+    "<summary>",
+    "</summary>",
+    '<thread thread_id="',
+    '<xref thread_id="',
+    '<href uri="',
+    "<custom>",
+    "</custom>",
+    '<layer value="',
+    '<src uri="',
+    "<tabular>",
+    "</tabular>",
+    '<checkbox class="unselected"/>',
+    '<checkbox class="selected"/>',
+    "<content>",
+    "</content>",
+    "<![CDATA[",
+    "]]>",
+    "<bold>",
+    "</bold>",
+    "<italic>",
+    "</italic>",
+    "<underline>",
+    "</underline>",
+    "<strikethrough>",
+    "</strikethrough>",
+    "<superscript>",
+    "</superscript>",
+    "<subscript>",
+    "</subscript>",
+    "<handwriting>",
+    "</handwriting>",
+    "<rtl>",
+    "</rtl>",
+    "<fcel/>",
+    "<ecel/>",
+    "<ched/>",
+    "<rhed/>",
+    "<corn/>",
+    "<srow/>",
+    "<lcel/>",
+    "<ucel/>",
+    "<xcel/>",
+    "<nl/>",
+    "<ldiv/>",
+    "<ldiv><marker>",
+    "</marker></ldiv>",
+)
+
+
+def _location_tokens(resolution: int) -> list[str]:
+    """Return one concrete ``<location value="N"/>`` token per value in ``[0, resolution)``."""
+    return [f'<location value="{value}"/>' for value in range(resolution)]
+
+
+def get_special_tokens(*, max_resolution: int = DEFAULT_MAX_RESOLUTION) -> list[str]:
+    """
+    Return the DocLang special-token vocabulary as a flat list of token strings.
+
+    This implements spec.md's "Token vocabulary" table, which is non-normative guidance
+    (the spec's "Recommendations" appendix) -- one reasonable tokenization, not a
+    conformance requirement other DocLang tools are bound to.
+
+    ``max_resolution`` should be the largest ``location@resolution`` (i.e.
+    ``default_resolution@width``/``@height``, or a per-``<location>`` override) used across your
+    documents; pass the larger of your x/y resolutions if they differ.
+    """
+    return [*_FIXED_TOKENS, *_location_tokens(max_resolution)]
diff --git a/spec.md b/spec.md
index e69cee5..f45c3fc 100644
--- a/spec.md
+++ b/spec.md
@@ -3286,8 +3286,6 @@ The token vocabulary trades off size and inference cost:
 | `<href uri="` | [`href`](#href) with `uri` attribute start |
 | `<custom>` | [`custom`](#custom) start |
 | `</custom>` | [`custom`](#custom) end |
-| `<smiles>` | custom SMILES element start (should best be [accordingly namespaced](#custom-vocabulary-naming-and-namespacing)) |
-| `</smiles>` | custom SMILES element end |
 | `<layer value="` | [`layer`](#layer) with `value` attribute start |
 | `<src uri="` | [`src`](#src) with `uri` attribute start |
 | `<tabular>` | [`tabular`](#tabular) start |
diff --git a/tests/test_tokenization.py b/tests/test_tokenization.py
new file mode 100644
index 0000000..5c5598a
--- /dev/null
+++ b/tests/test_tokenization.py
@@ -0,0 +1,46 @@
+"""Cross-check doclang.tokenization against the spec's "Token vocabulary" table."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from doclang.tokenization import DEFAULT_MAX_RESOLUTION, get_special_tokens
+
+SPEC_PATH = Path(__file__).resolve().parents[1] / "spec.md"
+
+_ROW_RE = re.compile(r"^\|(.+)\|(.+)\|\s*$", re.MULTILINE)
+_BACKTICK_RE = re.compile(r"`([^`]*)`")
+
+
+def _tokens_from_spec() -> list[str]:
+    text = SPEC_PATH.read_text()
+    section = text[text.index("#### Token vocabulary") : text.index("### Future Extensions")]
+
+    tokens: list[str] = []
+    for match in _ROW_RE.finditer(section):
+        column = match.group(1).strip()
+        if column in ("Token", "---") or not column.startswith("`"):
+            continue
+        if "..." in column:
+            # The location row lists a `<location value="0"/>` ... `<location value="511"/>` range.
+            tokens.extend(f'<location value="{value}"/>' for value in range(DEFAULT_MAX_RESOLUTION))
+            continue
+        tokens.extend(_BACKTICK_RE.findall(column))
+
+    return tokens
+
+
+def test_special_tokens_match_spec_vocabulary() -> None:
+    assert set(get_special_tokens()) == set(_tokens_from_spec())
+
+
+def test_special_tokens_have_no_duplicates() -> None:
+    tokens = get_special_tokens()
+    assert len(tokens) == len(set(tokens))
+
+
+def test_location_tokens_respect_max_resolution() -> None:
+    tokens = get_special_tokens(max_resolution=3)
+    location_tokens = [t for t in tokens if t.startswith('<location value="')]
+    assert location_tokens == ['<location value="0"/>', '<location value="1"/>', '<location value="2"/>']