From f232fcab5dc8fe94a9e74a32810952245167decf Mon Sep 17 00:00:00 2001 From: Kevin Castro Date: Thu, 5 Nov 2026 14:30:32 -0800 Subject: [PATCH 1/3] Test GIL release while parsing --- python/tests/test_parser.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/python/tests/test_parser.py b/python/tests/test_parser.py index ef13f68..19ccd76 100644 --- a/python/tests/test_parser.py +++ b/python/tests/test_parser.py @@ -1,3 +1,5 @@ +import sys + import markupever import pytest @@ -95,6 +97,38 @@ def test_parse_file_function(tmp_path): markupever.parse_file(file, markupever.HtmlOptions()) +@pytest.mark.parametrize( + "content, options", + [ + ("

text

" * 100_000, markupever.HtmlOptions()), + ("" + "text" * 100_000 + "", markupever.XmlOptions()), + ], + ids=["html", "xml"], +) +@pytest.mark.skipif(not getattr(sys, "_is_gil_enabled", lambda: True)(), reason="no GIL") +def test_parse_releases_gil(content, options): + import time + from concurrent.futures import ThreadPoolExecutor + + def parse(): + start = time.perf_counter() + markupever.parse(content, options) + return time.perf_counter() - start + + longest_pause = 0.0 + with ThreadPoolExecutor(1) as executor: + last = time.perf_counter() + future = executor.submit(parse) + while not future.done(): + now = time.perf_counter() + longest_pause = max(longest_pause, now - last) + last = now + longest_pause = max(longest_pause, time.perf_counter() - last) + + # This thread keeps running while the other one parses. + assert longest_pause < future.result() / 2 + + @pytest.mark.parametrize("kwargs", [{}, {"full_document": False}, {"fragment_context": "div"}]) def test_scripting_enabled_noscript(kwargs): html = "" From f4e5c99bfe5fa11e983a46fc0f1fd3f4906134af Mon Sep 17 00:00:00 2001 From: Kevin Castro Date: Thu, 5 Nov 2026 14:35:14 -0800 Subject: [PATCH 2/3] Release the GIL while parsing --- docs/docs/parser.md | 4 + python/markupever/_rustlib/__init__.pyi | 2 + python/markupever/parser.py | 29 +++--- src/lib.rs | 2 +- src/parser.rs | 130 +++++++++++++++--------- 5 files changed, 104 insertions(+), 63 deletions(-) diff --git a/docs/docs/parser.md b/docs/docs/parser.md index 471c1cb..f536b6b 100644 --- a/docs/docs/parser.md +++ b/docs/docs/parser.md @@ -45,6 +45,10 @@ We can use `.parse()` and `.parse_file()` functions to parse documents. the `.parse_file()` function gets a `BinaryIO`, a `TextIO` or a file path and parses it chunk by chunk; but `.parse()` function gets all document content at once. By this way, using `.parse_file()` is very better than `.parse()`. +!!! info "Threads" + + `.parse()` releases the GIL while parsing; `.parse_file()` and `Parser` do not. + Let's use them: === ".parse() function" diff --git a/python/markupever/_rustlib/__init__.pyi b/python/markupever/_rustlib/__init__.pyi index 9bb9ee7..316b83a 100644 --- a/python/markupever/_rustlib/__init__.pyi +++ b/python/markupever/_rustlib/__init__.pyi @@ -24,6 +24,7 @@ __all__ = [ "__version__", "_is_node_impl", "iter", + "parse", "serialize", ] @@ -342,6 +343,7 @@ class Select: def __iter__(self) -> Select: ... def __next__(self) -> Element: ... +def parse(content: str | bytes, options: HtmlOptions | XmlOptions) -> TreeDom: ... def serialize( node: _Node, indent: int = ..., include_self: bool = ..., is_html: bool | None = ... ) -> bytes: ... diff --git a/python/markupever/parser.py b/python/markupever/parser.py index eeba8f0..7b2049a 100644 --- a/python/markupever/parser.py +++ b/python/markupever/parser.py @@ -4,6 +4,19 @@ from .dom import TreeDom +def _options( + options: typing.Literal["html", "xml"] | _rustlib.HtmlOptions | _rustlib.XmlOptions, +) -> _rustlib.HtmlOptions | _rustlib.XmlOptions: + if isinstance(options, str): + if options == "html": + return _rustlib.HtmlOptions() + if options == "xml": + return _rustlib.XmlOptions() + raise ValueError(f"invalid parser options: {options!r}") + + return options + + class Parser: __slots__ = ("__raw", "__state") @@ -22,15 +35,7 @@ def __init__( for `options`, If your input is a HTML document, pass a `HtmlOptions`; If your input is a XML document, pass `XmlOptions`. """ - if isinstance(options, str): - if options == "html": - options = _rustlib.HtmlOptions() - elif options == "xml": - options = _rustlib.XmlOptions() - else: - raise ValueError(f"invalid parser options: {options!r}") - - self.__raw = _rustlib.Parser(options) + self.__raw = _rustlib.Parser(_options(options)) # 0 - processing # 1 - finished @@ -124,6 +129,8 @@ def parse( """ Parses HTML or XML content and returns the parsed document tree. + Releases the GIL while parsing. + Args: content: The HTML or XML content to parse, either as a string or bytes. options: Parsing options that specify whether to parse as HTML or XML. @@ -131,9 +138,7 @@ def parse( Returns: A TreeDom object representing the parsed document tree. """ - parser = Parser(options) - parser.process(content) - return parser.finish().into_dom() + return TreeDom(raw=_rustlib.parse(content, _options(options))) def parse_file( diff --git a/src/lib.rs b/src/lib.rs index cd777ec..78f5292 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -49,7 +49,7 @@ mod _rustlib { use crate::select::PySelect; #[pymodule_export] - use crate::parser::serialize; + use crate::parser::{parse, serialize}; #[pymodule_export] use crate::_is_node_impl; diff --git a/src/parser.rs b/src/parser.rs index dc310dd..6481b06 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2,7 +2,13 @@ /// /// # Note /// this type is immutable. -#[pyo3::pyclass(name = "HtmlOptions", module = "markupever._rustlib", frozen)] +#[pyo3::pyclass( + name = "HtmlOptions", + module = "markupever._rustlib", + frozen, + from_py_object +)] +#[derive(Clone)] pub struct PyHtmlOptions { exact_errors: bool, discard_bom: bool, @@ -149,7 +155,13 @@ impl PyHtmlOptions { } } -#[pyo3::pyclass(name = "XmlOptions", module = "markupever._rustlib", frozen)] +#[pyo3::pyclass( + name = "XmlOptions", + module = "markupever._rustlib", + frozen, + from_py_object +)] +#[derive(Clone)] pub struct PyXmlOptions { exact_errors: bool, discard_bom: bool, @@ -224,6 +236,45 @@ enum ParserState { } impl ParserState { + fn new(options: PyParserOptions) -> Self { + match options { + PyParserOptions::Html(options) => { + let tokenizer = treedom::html5ever::tokenizer::TokenizerOpts { + exact_errors: options.exact_errors, + discard_bom: options.discard_bom, + profile: options.profile, + ..Default::default() + }; + let tree_builder = treedom::html5ever::tree_builder::TreeBuilderOpts { + exact_errors: options.exact_errors, + iframe_srcdoc: options.iframe_srcdoc, + drop_doctype: options.drop_doctype, + quirks_mode: options.quirks_mode, + scripting_enabled: options.scripting_enabled, + }; + + Self::as_html(match options.fragment_context { + Some(context) => { + treedom::ParserSink::parse_html_fragment(context, tokenizer, tree_builder) + } + None => treedom::ParserSink::parse_html( + options.full_document, + tokenizer, + tree_builder, + ), + }) + } + PyParserOptions::Xml(options) => Self::as_xml(treedom::ParserSink::parse_xml( + treedom::xml5ever::tokenizer::XmlTokenizerOpts { + exact_errors: options.exact_errors, + discard_bom: options.discard_bom, + profile: options.profile, + ..Default::default() + }, + )), + } + } + fn as_html(val: treedom::html5ever::driver::Parser) -> Self { Self::OnHtml(Box::new(treedom::tendril::stream::Utf8LossyDecoder::new( val, @@ -275,7 +326,7 @@ impl std::fmt::Debug for ParserState { } #[derive(pyo3::FromPyObject)] -enum Input { +pub(crate) enum Input { Bytes(pyo3::pybacked::PyBackedBytes), Str(pyo3::pybacked::PyBackedStr), } @@ -290,9 +341,9 @@ pub struct PyParser { } #[derive(pyo3::FromPyObject)] -enum PyParserOptions<'p> { - Html(pyo3::PyRef<'p, PyHtmlOptions>), - Xml(pyo3::PyRef<'p, PyXmlOptions>), +pub(crate) enum PyParserOptions { + Html(PyHtmlOptions), + Xml(PyXmlOptions), } #[pyo3::pymethods] @@ -303,51 +354,8 @@ impl PyParser { /// If your input is a XML document, pass PyXmlOptions. #[new] fn new(options: PyParserOptions) -> pyo3::PyResult { - let state = { - match options { - PyParserOptions::Html(options) => { - let tokenizer = treedom::html5ever::tokenizer::TokenizerOpts { - exact_errors: options.exact_errors, - discard_bom: options.discard_bom, - profile: options.profile, - ..Default::default() - }; - let tree_builder = treedom::html5ever::tree_builder::TreeBuilderOpts { - exact_errors: options.exact_errors, - iframe_srcdoc: options.iframe_srcdoc, - drop_doctype: options.drop_doctype, - quirks_mode: options.quirks_mode, - scripting_enabled: options.scripting_enabled, - }; - - ParserState::as_html(match &options.fragment_context { - Some(context) => treedom::ParserSink::parse_html_fragment( - context.clone(), - tokenizer, - tree_builder, - ), - None => treedom::ParserSink::parse_html( - options.full_document, - tokenizer, - tree_builder, - ), - }) - } - PyParserOptions::Xml(options) => { - ParserState::as_xml(treedom::ParserSink::parse_xml( - treedom::xml5ever::tokenizer::XmlTokenizerOpts { - exact_errors: options.exact_errors, - discard_bom: options.discard_bom, - profile: options.profile, - ..Default::default() - }, - )) - } - } - }; - Ok(Self { - state: parking_lot::Mutex::new(state), + state: parking_lot::Mutex::new(ParserState::new(options)), }) } @@ -461,6 +469,28 @@ impl PyParser { } } +/// Parses `content` without holding the GIL. +#[pyo3::pyfunction] +pub(crate) fn parse( + py: pyo3::Python<'_>, + content: Input, + options: PyParserOptions, +) -> pyo3::PyResult { + let content = match &content { + Input::Bytes(b) => b, + Input::Str(s) => s.as_bytes(), + }; + + // html5ever's parser is not `Send`, so it is created and dropped inside `detach`. + let dom = py.detach(|| { + let mut state = ParserState::new(options); + state.process(content)?; + pyo3::PyResult::Ok(state.finish().into_dom()) + })?; + + Ok(super::tree::PyTreeDom::from_treedom(dom)) +} + #[pyo3::pyfunction] #[pyo3(signature=(node, indent=0, include_self=true, is_html=None))] pub fn serialize( From 2bac99689860d8c5321082c79a84c05f50fda958 Mon Sep 17 00:00:00 2001 From: Kevin Castro Date: Thu, 5 Nov 2026 14:47:01 -0800 Subject: [PATCH 3/3] Cover parsing without the GIL --- python/tests/test_parser.py | 51 +++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/python/tests/test_parser.py b/python/tests/test_parser.py index 19ccd76..f8d8a6d 100644 --- a/python/tests/test_parser.py +++ b/python/tests/test_parser.py @@ -129,6 +129,57 @@ def parse(): assert longest_pause < future.result() / 2 +@pytest.mark.parametrize( + "options", + [ + markupever.HtmlOptions(), + markupever.HtmlOptions(full_document=False, drop_doctype=True), + markupever.HtmlOptions(fragment_context="tbody"), + markupever.HtmlOptions(scripting_enabled=False), + markupever.XmlOptions(), + ], +) +@pytest.mark.parametrize( + "content", + [ + ( + "\ufeffCafé" + "
a & b
" + ), + b"

\xff invalid UTF-8

", + ], +) +def test_parse_matches_parser(options, content): + parser = markupever.Parser(options) + parser.process(content) + expected = parser.finish().into_dom().serialize() + + assert markupever.parse(content, options).serialize() == expected + + +def test_parse_invalid_arguments(): + with pytest.raises(TypeError): + markupever.parse(1) + + with pytest.raises(TypeError): + markupever.parse("", object()) + + with pytest.raises(ValueError): + markupever.parse("", "invalid") + + +def test_parse_in_threads(): + from concurrent.futures import ThreadPoolExecutor + + content = "
    " + "
  • item & more
  • " * 1_000 + "
" + expected = markupever.parse(content).serialize() + + with ThreadPoolExecutor(8) as executor: + results = list(executor.map(lambda _: markupever.parse(content).serialize(), range(64))) + + assert results == [expected] * 64 + + @pytest.mark.parametrize("kwargs", [{}, {"full_document": False}, {"fragment_context": "div"}]) def test_scripting_enabled_noscript(kwargs): html = ""