diff --git a/doclang/__init__.py b/doclang/__init__.py index 611bd2f..edcc478 100644 --- a/doclang/__init__.py +++ b/doclang/__init__.py @@ -2,6 +2,7 @@ from doclang.packaging import PackagingError, pack from doclang.schematron import SchematronBackendNotFound, SchematronValidator, SchematronViolation +from doclang.tokenization import get_special_tokens from doclang.validation import ValidationError, validate __all__ = [ @@ -10,6 +11,7 @@ "SchematronValidator", "SchematronViolation", "ValidationError", + "get_special_tokens", "pack", "validate", ] diff --git a/doclang/tokenization.py b/doclang/tokenization.py new file mode 100644 index 0000000..81d603a --- /dev/null +++ b/doclang/tokenization.py @@ -0,0 +1,128 @@ +"""DocLang special-token vocabulary, per spec.md's (non-normative) "Token vocabulary" recommendation.""" + +from __future__ import annotations + +DEFAULT_MAX_RESOLUTION = 512 + +_HEADING_LEVELS = range(2, 7) +_FIELD_HEADING_LEVELS = range(2, 7) + +_FIXED_TOKENS: tuple[str, ...] = ( + "", + "", + "", + '"/>', + "", + "", + "", + *(f'' for level in _HEADING_LEVELS), + "", + "", + "", + "", + "", + "", + "", + "", + "", + "", + '', + "", + "", + "
", + "", + "", + "", + "", + '