Try to fix bad JSON due to unescaped double quotes

extruct/jsonld.py

@@ -3,15 +3,9 @@
 JSON-LD extractor
 """
 
-import json
-import re
-
-import jstyleson
 import lxml.etree
 
-from extruct.utils import parse_html
-
-HTML_OR_JS_COMMENTLINE = re.compile(r"^\s*(//.*|<!--.*-->)")
+from extruct.utils import _parse_json, parse_html
 
 
 class JsonLdExtractor:
@@ -36,12 +30,7 @@ def _extract_items(self, node):
         script = node.xpath("string()").strip()
         if not script:
             return
-        try:
-            # TODO: `strict=False` can be configurable if needed
-            data = json.loads(script, strict=False)
-        except ValueError:
-            # sometimes JSON-decoding errors are due to leading HTML or JavaScript comments
-            data = jstyleson.loads(HTML_OR_JS_COMMENTLINE.sub("", script), strict=False)
+        data = _parse_json(script)
         if isinstance(data, list):
             yield from data
         elif isinstance(data, dict):

extruct/utils.py

@@ -1,4 +1,8 @@
 # mypy: disallow_untyped_defs=False
+import json
+import re
+
+import jstyleson
 import lxml.html
 
 from extruct.xmldom import XmlDomHTMLParser
@@ -10,6 +14,40 @@ def parse_html(html, encoding):
     return lxml.html.fromstring(html, parser=parser)
 
 
+_HTML_COMMENTLINE = re.compile(r"^\s*<!--.*?-->")
+
+
+def _parse_json(json_string):
+    try:
+        return json.loads(json_string, strict=False)
+    except ValueError:
+        pass
+
+    # Comments are stripped once, up front: the error offsets used below must
+    # refer to the same string that json.loads() reads. jstyleson.dispose()
+    # handles JavaScript comments and trailing commas, but not HTML comments.
+    json_string = jstyleson.dispose(_HTML_COMMENTLINE.sub("", json_string))
+
+    # Each iteration escapes one quote and never adds one, so the loop runs at
+    # most as many times as there are quotes in the string.
+    while True:
+        try:
+            return json.loads(json_string, strict=False)
+        except json.JSONDecodeError as error:
+            # An unescaped double quote inside a string value ends that value
+            # early, so the parser finds text where it expects the next item.
+            # Escape the quote that ended the value and try again. The reported
+            # position is past any whitespace that follows the quote.
+            quote = json_string.rfind('"', 0, error.pos)
+            if (
+                error.msg != "Expecting ',' delimiter"
+                or quote < 0
+                or json_string[quote + 1 : error.pos].strip()
+            ):
+                raise
+            json_string = json_string[:quote] + "\\" + json_string[quote:]
+
+
 def parse_xmldom_html(html, encoding):
     """Parse HTML using XmlDomHTMLParser, return a tree"""
     parser = XmlDomHTMLParser(encoding=encoding)

tests/test_utils.py

@@ -0,0 +1,40 @@
+# mypy: disallow_untyped_defs=False
+from pytest import mark, raises
+
+from extruct.utils import _parse_json
+
+
+@mark.parametrize(
+    "input,output",
+    [
+        # unescaped quotes
+        ('{"a": ["10\'5""]}', {"a": ["10'5\""]}),
+        ('{"a": ["Say "Hello""]}', {"a": ['Say "Hello"']}),
+        ('{"a": "Say "Hello" there"}', {"a": 'Say "Hello" there'}),
+        ('{"a": "1. two "buttons".5. Dab!"}', {"a": '1. two "buttons".5. Dab!'}),
+        # unescaped quotes combined with what the comment stripping handles
+        ('{"a": [1,], "b": ["Say "Hello""]}', {"a": [1], "b": ['Say "Hello"']}),
+        ('{\n// note\n"b": ["Say "Hello""]}', {"b": ['Say "Hello"']}),
+        ('<!-- note -->{"a": "Say "Hello""}', {"a": 'Say "Hello"'}),
+        # a missing comma is indistinguishable from an unescaped quote, and is
+        # resolved by merging the values
+        ('{"a": ["x" "y"]}', {"a": ['x" "y']}),
+    ],
+)
+def test_parse_json(input, output):
+    assert _parse_json(input) == output
+
+
+@mark.parametrize(
+    "input",
+    [
+        "",
+        "not json",
+        "{",
+        '{"a": [}',
+        '{"a": "b"',
+    ],
+)
+def test_parse_json_unfixable(input):
+    with raises(ValueError):
+        _parse_json(input)