diff --git a/extruct/jsonld.py b/extruct/jsonld.py index d25a418..225e691 100644 --- a/extruct/jsonld.py +++ b/extruct/jsonld.py @@ -3,15 +3,9 @@ JSON-LD extractor """ -import json -import re - -import jstyleson import lxml.etree -from extruct.utils import parse_html - -HTML_OR_JS_COMMENTLINE = re.compile(r"^\s*(//.*|)") +from extruct.utils import _parse_json, parse_html class JsonLdExtractor: @@ -36,12 +30,7 @@ def _extract_items(self, node): script = node.xpath("string()").strip() if not script: return - try: - # TODO: `strict=False` can be configurable if needed - data = json.loads(script, strict=False) - except ValueError: - # sometimes JSON-decoding errors are due to leading HTML or JavaScript comments - data = jstyleson.loads(HTML_OR_JS_COMMENTLINE.sub("", script), strict=False) + data = _parse_json(script) if isinstance(data, list): yield from data elif isinstance(data, dict): diff --git a/extruct/utils.py b/extruct/utils.py index 6dc9810..af6b562 100644 --- a/extruct/utils.py +++ b/extruct/utils.py @@ -1,4 +1,8 @@ # mypy: disallow_untyped_defs=False +import json +import re + +import jstyleson import lxml.html from extruct.xmldom import XmlDomHTMLParser @@ -10,6 +14,40 @@ def parse_html(html, encoding): return lxml.html.fromstring(html, parser=parser) +_HTML_COMMENTLINE = re.compile(r"^\s*") + + +def _parse_json(json_string): + try: + return json.loads(json_string, strict=False) + except ValueError: + pass + + # Comments are stripped once, up front: the error offsets used below must + # refer to the same string that json.loads() reads. jstyleson.dispose() + # handles JavaScript comments and trailing commas, but not HTML comments. + json_string = jstyleson.dispose(_HTML_COMMENTLINE.sub("", json_string)) + + # Each iteration escapes one quote and never adds one, so the loop runs at + # most as many times as there are quotes in the string. + while True: + try: + return json.loads(json_string, strict=False) + except json.JSONDecodeError as error: + # An unescaped double quote inside a string value ends that value + # early, so the parser finds text where it expects the next item. + # Escape the quote that ended the value and try again. The reported + # position is past any whitespace that follows the quote. + quote = json_string.rfind('"', 0, error.pos) + if ( + error.msg != "Expecting ',' delimiter" + or quote < 0 + or json_string[quote + 1 : error.pos].strip() + ): + raise + json_string = json_string[:quote] + "\\" + json_string[quote:] + + def parse_xmldom_html(html, encoding): """Parse HTML using XmlDomHTMLParser, return a tree""" parser = XmlDomHTMLParser(encoding=encoding) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..0597abe --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,40 @@ +# mypy: disallow_untyped_defs=False +from pytest import mark, raises + +from extruct.utils import _parse_json + + +@mark.parametrize( + "input,output", + [ + # unescaped quotes + ('{"a": ["10\'5""]}', {"a": ["10'5\""]}), + ('{"a": ["Say "Hello""]}', {"a": ['Say "Hello"']}), + ('{"a": "Say "Hello" there"}', {"a": 'Say "Hello" there'}), + ('{"a": "1. two "buttons".5. Dab!"}', {"a": '1. two "buttons".5. Dab!'}), + # unescaped quotes combined with what the comment stripping handles + ('{"a": [1,], "b": ["Say "Hello""]}', {"a": [1], "b": ['Say "Hello"']}), + ('{\n// note\n"b": ["Say "Hello""]}', {"b": ['Say "Hello"']}), + ('{"a": "Say "Hello""}', {"a": 'Say "Hello"'}), + # a missing comma is indistinguishable from an unescaped quote, and is + # resolved by merging the values + ('{"a": ["x" "y"]}', {"a": ['x" "y']}), + ], +) +def test_parse_json(input, output): + assert _parse_json(input) == output + + +@mark.parametrize( + "input", + [ + "", + "not json", + "{", + '{"a": [}', + '{"a": "b"', + ], +) +def test_parse_json_unfixable(input): + with raises(ValueError): + _parse_json(input)