diff --git a/docs/docs/parser.md b/docs/docs/parser.md
index 471c1cb..f536b6b 100644
--- a/docs/docs/parser.md
+++ b/docs/docs/parser.md
@@ -45,6 +45,10 @@ We can use `.parse()` and `.parse_file()` functions to parse documents.
the `.parse_file()` function gets a `BinaryIO`, a `TextIO` or a file path and parses it chunk by chunk; but `.parse()` function gets all document content at once. By this way, using `.parse_file()` is very better than `.parse()`.
+!!! info "Threads"
+
+ `.parse()` releases the GIL while parsing; `.parse_file()` and `Parser` do not.
+
Let's use them:
=== ".parse() function"
diff --git a/python/markupever/_rustlib/__init__.pyi b/python/markupever/_rustlib/__init__.pyi
index 9bb9ee7..316b83a 100644
--- a/python/markupever/_rustlib/__init__.pyi
+++ b/python/markupever/_rustlib/__init__.pyi
@@ -24,6 +24,7 @@ __all__ = [
"__version__",
"_is_node_impl",
"iter",
+ "parse",
"serialize",
]
@@ -342,6 +343,7 @@ class Select:
def __iter__(self) -> Select: ...
def __next__(self) -> Element: ...
+def parse(content: str | bytes, options: HtmlOptions | XmlOptions) -> TreeDom: ...
def serialize(
node: _Node, indent: int = ..., include_self: bool = ..., is_html: bool | None = ...
) -> bytes: ...
diff --git a/python/markupever/parser.py b/python/markupever/parser.py
index eeba8f0..7b2049a 100644
--- a/python/markupever/parser.py
+++ b/python/markupever/parser.py
@@ -4,6 +4,19 @@
from .dom import TreeDom
+def _options(
+ options: typing.Literal["html", "xml"] | _rustlib.HtmlOptions | _rustlib.XmlOptions,
+) -> _rustlib.HtmlOptions | _rustlib.XmlOptions:
+ if isinstance(options, str):
+ if options == "html":
+ return _rustlib.HtmlOptions()
+ if options == "xml":
+ return _rustlib.XmlOptions()
+ raise ValueError(f"invalid parser options: {options!r}")
+
+ return options
+
+
class Parser:
__slots__ = ("__raw", "__state")
@@ -22,15 +35,7 @@ def __init__(
for `options`, If your input is a HTML document, pass a `HtmlOptions`;
If your input is a XML document, pass `XmlOptions`.
"""
- if isinstance(options, str):
- if options == "html":
- options = _rustlib.HtmlOptions()
- elif options == "xml":
- options = _rustlib.XmlOptions()
- else:
- raise ValueError(f"invalid parser options: {options!r}")
-
- self.__raw = _rustlib.Parser(options)
+ self.__raw = _rustlib.Parser(_options(options))
# 0 - processing
# 1 - finished
@@ -124,6 +129,8 @@ def parse(
"""
Parses HTML or XML content and returns the parsed document tree.
+ Releases the GIL while parsing.
+
Args:
content: The HTML or XML content to parse, either as a string or bytes.
options: Parsing options that specify whether to parse as HTML or XML.
@@ -131,9 +138,7 @@ def parse(
Returns:
A TreeDom object representing the parsed document tree.
"""
- parser = Parser(options)
- parser.process(content)
- return parser.finish().into_dom()
+ return TreeDom(raw=_rustlib.parse(content, _options(options)))
def parse_file(
diff --git a/python/tests/test_parser.py b/python/tests/test_parser.py
index ef13f68..f8d8a6d 100644
--- a/python/tests/test_parser.py
+++ b/python/tests/test_parser.py
@@ -1,3 +1,5 @@
+import sys
+
import markupever
import pytest
@@ -95,6 +97,89 @@ def test_parse_file_function(tmp_path):
markupever.parse_file(file, markupever.HtmlOptions())
+@pytest.mark.parametrize(
+ "content, options",
+ [
+ ("
text
" * 100_000, markupever.HtmlOptions()),
+ ("" + "text" * 100_000 + "", markupever.XmlOptions()),
+ ],
+ ids=["html", "xml"],
+)
+@pytest.mark.skipif(not getattr(sys, "_is_gil_enabled", lambda: True)(), reason="no GIL")
+def test_parse_releases_gil(content, options):
+ import time
+ from concurrent.futures import ThreadPoolExecutor
+
+ def parse():
+ start = time.perf_counter()
+ markupever.parse(content, options)
+ return time.perf_counter() - start
+
+ longest_pause = 0.0
+ with ThreadPoolExecutor(1) as executor:
+ last = time.perf_counter()
+ future = executor.submit(parse)
+ while not future.done():
+ now = time.perf_counter()
+ longest_pause = max(longest_pause, now - last)
+ last = now
+ longest_pause = max(longest_pause, time.perf_counter() - last)
+
+ # This thread keeps running while the other one parses.
+ assert longest_pause < future.result() / 2
+
+
+@pytest.mark.parametrize(
+ "options",
+ [
+ markupever.HtmlOptions(),
+ markupever.HtmlOptions(full_document=False, drop_doctype=True),
+ markupever.HtmlOptions(fragment_context="tbody"),
+ markupever.HtmlOptions(scripting_enabled=False),
+ markupever.XmlOptions(),
+ ],
+)
+@pytest.mark.parametrize(
+ "content",
+ [
+ (
+ "\ufeffCafé"
+ ""
+ ),
+ b"\xff invalid UTF-8
",
+ ],
+)
+def test_parse_matches_parser(options, content):
+ parser = markupever.Parser(options)
+ parser.process(content)
+ expected = parser.finish().into_dom().serialize()
+
+ assert markupever.parse(content, options).serialize() == expected
+
+
+def test_parse_invalid_arguments():
+ with pytest.raises(TypeError):
+ markupever.parse(1)
+
+ with pytest.raises(TypeError):
+ markupever.parse("", object())
+
+ with pytest.raises(ValueError):
+ markupever.parse("", "invalid")
+
+
+def test_parse_in_threads():
+ from concurrent.futures import ThreadPoolExecutor
+
+ content = "" + "- item & more
" * 1_000 + "
"
+ expected = markupever.parse(content).serialize()
+
+ with ThreadPoolExecutor(8) as executor:
+ results = list(executor.map(lambda _: markupever.parse(content).serialize(), range(64)))
+
+ assert results == [expected] * 64
+
+
@pytest.mark.parametrize("kwargs", [{}, {"full_document": False}, {"fragment_context": "div"}])
def test_scripting_enabled_noscript(kwargs):
html = ""
diff --git a/src/lib.rs b/src/lib.rs
index cd777ec..78f5292 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -49,7 +49,7 @@ mod _rustlib {
use crate::select::PySelect;
#[pymodule_export]
- use crate::parser::serialize;
+ use crate::parser::{parse, serialize};
#[pymodule_export]
use crate::_is_node_impl;
diff --git a/src/parser.rs b/src/parser.rs
index dc310dd..6481b06 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -2,7 +2,13 @@
///
/// # Note
/// this type is immutable.
-#[pyo3::pyclass(name = "HtmlOptions", module = "markupever._rustlib", frozen)]
+#[pyo3::pyclass(
+ name = "HtmlOptions",
+ module = "markupever._rustlib",
+ frozen,
+ from_py_object
+)]
+#[derive(Clone)]
pub struct PyHtmlOptions {
exact_errors: bool,
discard_bom: bool,
@@ -149,7 +155,13 @@ impl PyHtmlOptions {
}
}
-#[pyo3::pyclass(name = "XmlOptions", module = "markupever._rustlib", frozen)]
+#[pyo3::pyclass(
+ name = "XmlOptions",
+ module = "markupever._rustlib",
+ frozen,
+ from_py_object
+)]
+#[derive(Clone)]
pub struct PyXmlOptions {
exact_errors: bool,
discard_bom: bool,
@@ -224,6 +236,45 @@ enum ParserState {
}
impl ParserState {
+ fn new(options: PyParserOptions) -> Self {
+ match options {
+ PyParserOptions::Html(options) => {
+ let tokenizer = treedom::html5ever::tokenizer::TokenizerOpts {
+ exact_errors: options.exact_errors,
+ discard_bom: options.discard_bom,
+ profile: options.profile,
+ ..Default::default()
+ };
+ let tree_builder = treedom::html5ever::tree_builder::TreeBuilderOpts {
+ exact_errors: options.exact_errors,
+ iframe_srcdoc: options.iframe_srcdoc,
+ drop_doctype: options.drop_doctype,
+ quirks_mode: options.quirks_mode,
+ scripting_enabled: options.scripting_enabled,
+ };
+
+ Self::as_html(match options.fragment_context {
+ Some(context) => {
+ treedom::ParserSink::parse_html_fragment(context, tokenizer, tree_builder)
+ }
+ None => treedom::ParserSink::parse_html(
+ options.full_document,
+ tokenizer,
+ tree_builder,
+ ),
+ })
+ }
+ PyParserOptions::Xml(options) => Self::as_xml(treedom::ParserSink::parse_xml(
+ treedom::xml5ever::tokenizer::XmlTokenizerOpts {
+ exact_errors: options.exact_errors,
+ discard_bom: options.discard_bom,
+ profile: options.profile,
+ ..Default::default()
+ },
+ )),
+ }
+ }
+
fn as_html(val: treedom::html5ever::driver::Parser) -> Self {
Self::OnHtml(Box::new(treedom::tendril::stream::Utf8LossyDecoder::new(
val,
@@ -275,7 +326,7 @@ impl std::fmt::Debug for ParserState {
}
#[derive(pyo3::FromPyObject)]
-enum Input {
+pub(crate) enum Input {
Bytes(pyo3::pybacked::PyBackedBytes),
Str(pyo3::pybacked::PyBackedStr),
}
@@ -290,9 +341,9 @@ pub struct PyParser {
}
#[derive(pyo3::FromPyObject)]
-enum PyParserOptions<'p> {
- Html(pyo3::PyRef<'p, PyHtmlOptions>),
- Xml(pyo3::PyRef<'p, PyXmlOptions>),
+pub(crate) enum PyParserOptions {
+ Html(PyHtmlOptions),
+ Xml(PyXmlOptions),
}
#[pyo3::pymethods]
@@ -303,51 +354,8 @@ impl PyParser {
/// If your input is a XML document, pass PyXmlOptions.
#[new]
fn new(options: PyParserOptions) -> pyo3::PyResult {
- let state = {
- match options {
- PyParserOptions::Html(options) => {
- let tokenizer = treedom::html5ever::tokenizer::TokenizerOpts {
- exact_errors: options.exact_errors,
- discard_bom: options.discard_bom,
- profile: options.profile,
- ..Default::default()
- };
- let tree_builder = treedom::html5ever::tree_builder::TreeBuilderOpts {
- exact_errors: options.exact_errors,
- iframe_srcdoc: options.iframe_srcdoc,
- drop_doctype: options.drop_doctype,
- quirks_mode: options.quirks_mode,
- scripting_enabled: options.scripting_enabled,
- };
-
- ParserState::as_html(match &options.fragment_context {
- Some(context) => treedom::ParserSink::parse_html_fragment(
- context.clone(),
- tokenizer,
- tree_builder,
- ),
- None => treedom::ParserSink::parse_html(
- options.full_document,
- tokenizer,
- tree_builder,
- ),
- })
- }
- PyParserOptions::Xml(options) => {
- ParserState::as_xml(treedom::ParserSink::parse_xml(
- treedom::xml5ever::tokenizer::XmlTokenizerOpts {
- exact_errors: options.exact_errors,
- discard_bom: options.discard_bom,
- profile: options.profile,
- ..Default::default()
- },
- ))
- }
- }
- };
-
Ok(Self {
- state: parking_lot::Mutex::new(state),
+ state: parking_lot::Mutex::new(ParserState::new(options)),
})
}
@@ -461,6 +469,28 @@ impl PyParser {
}
}
+/// Parses `content` without holding the GIL.
+#[pyo3::pyfunction]
+pub(crate) fn parse(
+ py: pyo3::Python<'_>,
+ content: Input,
+ options: PyParserOptions,
+) -> pyo3::PyResult {
+ let content = match &content {
+ Input::Bytes(b) => b,
+ Input::Str(s) => s.as_bytes(),
+ };
+
+ // html5ever's parser is not `Send`, so it is created and dropped inside `detach`.
+ let dom = py.detach(|| {
+ let mut state = ParserState::new(options);
+ state.process(content)?;
+ pyo3::PyResult::Ok(state.finish().into_dom())
+ })?;
+
+ Ok(super::tree::PyTreeDom::from_treedom(dom))
+}
+
#[pyo3::pyfunction]
#[pyo3(signature=(node, indent=0, include_self=true, is_html=None))]
pub fn serialize(