Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/docs/parser.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ We can use `.parse()` and `.parse_file()` functions to parse documents.

the `.parse_file()` function gets a `BinaryIO`, a `TextIO` or a file path and parses it chunk by chunk; but `.parse()` function gets all document content at once. By this way, using `.parse_file()` is very better than `.parse()`.

!!! info "Threads"

`.parse()` releases the GIL while parsing; `.parse_file()` and `Parser` do not.

Let's use them:

=== ".parse() function"
Expand Down
2 changes: 2 additions & 0 deletions python/markupever/_rustlib/__init__.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ __all__ = [
"__version__",
"_is_node_impl",
"iter",
"parse",
"serialize",
]

Expand Down Expand Up @@ -342,6 +343,7 @@ class Select:
def __iter__(self) -> Select: ...
def __next__(self) -> Element: ...

def parse(content: str | bytes, options: HtmlOptions | XmlOptions) -> TreeDom: ...
def serialize(
node: _Node, indent: int = ..., include_self: bool = ..., is_html: bool | None = ...
) -> bytes: ...
Expand Down
29 changes: 17 additions & 12 deletions python/markupever/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,19 @@
from .dom import TreeDom


def _options(
options: typing.Literal["html", "xml"] | _rustlib.HtmlOptions | _rustlib.XmlOptions,
) -> _rustlib.HtmlOptions | _rustlib.XmlOptions:
if isinstance(options, str):
if options == "html":
return _rustlib.HtmlOptions()
if options == "xml":
return _rustlib.XmlOptions()
raise ValueError(f"invalid parser options: {options!r}")

return options


class Parser:
__slots__ = ("__raw", "__state")

Expand All @@ -22,15 +35,7 @@ def __init__(
for `options`, If your input is a HTML document, pass a `HtmlOptions`;
If your input is a XML document, pass `XmlOptions`.
"""
if isinstance(options, str):
if options == "html":
options = _rustlib.HtmlOptions()
elif options == "xml":
options = _rustlib.XmlOptions()
else:
raise ValueError(f"invalid parser options: {options!r}")

self.__raw = _rustlib.Parser(options)
self.__raw = _rustlib.Parser(_options(options))

# 0 - processing
# 1 - finished
Expand Down Expand Up @@ -124,16 +129,16 @@ def parse(
"""
Parses HTML or XML content and returns the parsed document tree.

Releases the GIL while parsing.

Args:
content: The HTML or XML content to parse, either as a string or bytes.
options: Parsing options that specify whether to parse as HTML or XML.

Returns:
A TreeDom object representing the parsed document tree.
"""
parser = Parser(options)
parser.process(content)
return parser.finish().into_dom()
return TreeDom(raw=_rustlib.parse(content, _options(options)))


def parse_file(
Expand Down
85 changes: 85 additions & 0 deletions python/tests/test_parser.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import sys

import markupever
import pytest

Expand Down Expand Up @@ -95,6 +97,89 @@ def test_parse_file_function(tmp_path):
markupever.parse_file(file, markupever.HtmlOptions())


@pytest.mark.parametrize(
"content, options",
[
("<p>text</p>" * 100_000, markupever.HtmlOptions()),
("<root>" + "<a>text</a>" * 100_000 + "</root>", markupever.XmlOptions()),
],
ids=["html", "xml"],
)
@pytest.mark.skipif(not getattr(sys, "_is_gil_enabled", lambda: True)(), reason="no GIL")
def test_parse_releases_gil(content, options):
import time
from concurrent.futures import ThreadPoolExecutor

def parse():
start = time.perf_counter()
markupever.parse(content, options)
return time.perf_counter() - start

longest_pause = 0.0
with ThreadPoolExecutor(1) as executor:
last = time.perf_counter()
future = executor.submit(parse)
while not future.done():
now = time.perf_counter()
longest_pause = max(longest_pause, now - last)
last = now
longest_pause = max(longest_pause, time.perf_counter() - last)

# This thread keeps running while the other one parses.
assert longest_pause < future.result() / 2


@pytest.mark.parametrize(
"options",
[
markupever.HtmlOptions(),
markupever.HtmlOptions(full_document=False, drop_doctype=True),
markupever.HtmlOptions(fragment_context="tbody"),
markupever.HtmlOptions(scripting_enabled=False),
markupever.XmlOptions(),
],
)
@pytest.mark.parametrize(
"content",
[
(
"\ufeff<!DOCTYPE html><title>Café</title><noscript><b>1234</b></noscript>"
"<table><tr><td>a &amp; b<!-- c --></table><svg><template>x</template></svg>"
),
b"<p>\xff invalid UTF-8</p>",
],
)
def test_parse_matches_parser(options, content):
parser = markupever.Parser(options)
parser.process(content)
expected = parser.finish().into_dom().serialize()

assert markupever.parse(content, options).serialize() == expected


def test_parse_invalid_arguments():
with pytest.raises(TypeError):
markupever.parse(1)

with pytest.raises(TypeError):
markupever.parse("", object())

with pytest.raises(ValueError):
markupever.parse("", "invalid")


def test_parse_in_threads():
from concurrent.futures import ThreadPoolExecutor

content = "<ul>" + "<li>item &amp; more</li>" * 1_000 + "</ul>"
expected = markupever.parse(content).serialize()

with ThreadPoolExecutor(8) as executor:
results = list(executor.map(lambda _: markupever.parse(content).serialize(), range(64)))

assert results == [expected] * 64


@pytest.mark.parametrize("kwargs", [{}, {"full_document": False}, {"fragment_context": "div"}])
def test_scripting_enabled_noscript(kwargs):
html = "<body><noscript><p>Code <strong>1234</strong> &amp; more</p></noscript></body>"
Expand Down
2 changes: 1 addition & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ mod _rustlib {
use crate::select::PySelect;

#[pymodule_export]
use crate::parser::serialize;
use crate::parser::{parse, serialize};

#[pymodule_export]
use crate::_is_node_impl;
Expand Down
130 changes: 80 additions & 50 deletions src/parser.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,13 @@
///
/// # Note
/// this type is immutable.
#[pyo3::pyclass(name = "HtmlOptions", module = "markupever._rustlib", frozen)]
#[pyo3::pyclass(
name = "HtmlOptions",
module = "markupever._rustlib",
frozen,
from_py_object
)]
#[derive(Clone)]
pub struct PyHtmlOptions {
exact_errors: bool,
discard_bom: bool,
Expand Down Expand Up @@ -149,7 +155,13 @@ impl PyHtmlOptions {
}
}

#[pyo3::pyclass(name = "XmlOptions", module = "markupever._rustlib", frozen)]
#[pyo3::pyclass(
name = "XmlOptions",
module = "markupever._rustlib",
frozen,
from_py_object
)]
#[derive(Clone)]
pub struct PyXmlOptions {
exact_errors: bool,
discard_bom: bool,
Expand Down Expand Up @@ -224,6 +236,45 @@ enum ParserState {
}

impl ParserState {
fn new(options: PyParserOptions) -> Self {
match options {
PyParserOptions::Html(options) => {
let tokenizer = treedom::html5ever::tokenizer::TokenizerOpts {
exact_errors: options.exact_errors,
discard_bom: options.discard_bom,
profile: options.profile,
..Default::default()
};
let tree_builder = treedom::html5ever::tree_builder::TreeBuilderOpts {
exact_errors: options.exact_errors,
iframe_srcdoc: options.iframe_srcdoc,
drop_doctype: options.drop_doctype,
quirks_mode: options.quirks_mode,
scripting_enabled: options.scripting_enabled,
};

Self::as_html(match options.fragment_context {
Some(context) => {
treedom::ParserSink::parse_html_fragment(context, tokenizer, tree_builder)
}
None => treedom::ParserSink::parse_html(
options.full_document,
tokenizer,
tree_builder,
),
})
}
PyParserOptions::Xml(options) => Self::as_xml(treedom::ParserSink::parse_xml(
treedom::xml5ever::tokenizer::XmlTokenizerOpts {
exact_errors: options.exact_errors,
discard_bom: options.discard_bom,
profile: options.profile,
..Default::default()
},
)),
}
}

fn as_html(val: treedom::html5ever::driver::Parser<treedom::ParserSink>) -> Self {
Self::OnHtml(Box::new(treedom::tendril::stream::Utf8LossyDecoder::new(
val,
Expand Down Expand Up @@ -275,7 +326,7 @@ impl std::fmt::Debug for ParserState {
}

#[derive(pyo3::FromPyObject)]
enum Input {
pub(crate) enum Input {
Bytes(pyo3::pybacked::PyBackedBytes),
Str(pyo3::pybacked::PyBackedStr),
}
Expand All @@ -290,9 +341,9 @@ pub struct PyParser {
}

#[derive(pyo3::FromPyObject)]
enum PyParserOptions<'p> {
Html(pyo3::PyRef<'p, PyHtmlOptions>),
Xml(pyo3::PyRef<'p, PyXmlOptions>),
pub(crate) enum PyParserOptions {
Html(PyHtmlOptions),
Xml(PyXmlOptions),
}

#[pyo3::pymethods]
Expand All @@ -303,51 +354,8 @@ impl PyParser {
/// If your input is a XML document, pass PyXmlOptions.
#[new]
fn new(options: PyParserOptions) -> pyo3::PyResult<Self> {
let state = {
match options {
PyParserOptions::Html(options) => {
let tokenizer = treedom::html5ever::tokenizer::TokenizerOpts {
exact_errors: options.exact_errors,
discard_bom: options.discard_bom,
profile: options.profile,
..Default::default()
};
let tree_builder = treedom::html5ever::tree_builder::TreeBuilderOpts {
exact_errors: options.exact_errors,
iframe_srcdoc: options.iframe_srcdoc,
drop_doctype: options.drop_doctype,
quirks_mode: options.quirks_mode,
scripting_enabled: options.scripting_enabled,
};

ParserState::as_html(match &options.fragment_context {
Some(context) => treedom::ParserSink::parse_html_fragment(
context.clone(),
tokenizer,
tree_builder,
),
None => treedom::ParserSink::parse_html(
options.full_document,
tokenizer,
tree_builder,
),
})
}
PyParserOptions::Xml(options) => {
ParserState::as_xml(treedom::ParserSink::parse_xml(
treedom::xml5ever::tokenizer::XmlTokenizerOpts {
exact_errors: options.exact_errors,
discard_bom: options.discard_bom,
profile: options.profile,
..Default::default()
},
))
}
}
};

Ok(Self {
state: parking_lot::Mutex::new(state),
state: parking_lot::Mutex::new(ParserState::new(options)),
})
}

Expand Down Expand Up @@ -461,6 +469,28 @@ impl PyParser {
}
}

/// Parses `content` without holding the GIL.
#[pyo3::pyfunction]
pub(crate) fn parse(
py: pyo3::Python<'_>,
content: Input,
options: PyParserOptions,
) -> pyo3::PyResult<super::tree::PyTreeDom> {
let content = match &content {
Input::Bytes(b) => b,
Input::Str(s) => s.as_bytes(),
};

// html5ever's parser is not `Send`, so it is created and dropped inside `detach`.
let dom = py.detach(|| {
let mut state = ParserState::new(options);
state.process(content)?;
pyo3::PyResult::Ok(state.finish().into_dom())
})?;

Ok(super::tree::PyTreeDom::from_treedom(dom))
}

#[pyo3::pyfunction]
#[pyo3(signature=(node, indent=0, include_self=true, is_html=None))]
pub fn serialize(
Expand Down
Loading