diff --git a/README.rst b/README.rst
index 01595119..e0891a4f 100644
--- a/README.rst
+++ b/README.rst
@@ -143,9 +143,7 @@ First fetch the HTML using python-requests and then feed the response body to ``
'https://www.optimizesmart.com/wp-content/uploads/2010/07/open-graph-protocol.jpg'),
( 'og:image:secure_url',
'https://www.optimizesmart.com/wp-content/uploads/2010/07/open-graph-protocol.jpg')]}],
- 'rdfa': [ { '@id': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/#header',
- 'http://www.w3.org/1999/xhtml/vocab#role': [ { '@id': 'http://www.w3.org/1999/xhtml/vocab#banner'}]},
- { '@id': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/',
+ 'rdfa': [ { '@id': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/',
'article:modified_time': [ { '@value': '2018-03-09T16:26:35+00:00'}],
'article:published_time': [ { '@value': '2010-07-02T18:57:23+00:00'}],
'article:publisher': [ { '@value': 'https://www.facebook.com/optimizesmart/'}],
@@ -171,7 +169,9 @@ First fetch the HTML using python-requests and then feed the response body to ``
'http://ogp.me/ns#type': [{'@value': 'article'}],
'http://ogp.me/ns#updated_time': [ { '@value': '2018-03-09T16:26:35+00:00'}],
'http://ogp.me/ns#url': [ { '@value': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/'}],
- 'https://api.w.org/': [ { '@id': 'https://www.optimizesmart.com/wp-json/'}]}]}
+ 'https://api.w.org/': [ { '@id': 'https://www.optimizesmart.com/wp-json/'}]},
+ { '@id': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/#header',
+ 'http://www.w3.org/1999/xhtml/vocab#role': [ { '@id': 'http://www.w3.org/1999/xhtml/vocab#banner'}]}]}
Select syntaxes
+++++++++++++++
diff --git a/extruct/rdfa.py b/extruct/rdfa.py
index d84bd595..afeef91f 100644
--- a/extruct/rdfa.py
+++ b/extruct/rdfa.py
@@ -8,6 +8,7 @@
import logging
import re
from collections import defaultdict
+from functools import partial
rdflib_logger = logging.getLogger("rdflib")
rdflib_logger.setLevel(logging.ERROR)
@@ -110,13 +111,28 @@ def _sort(self, unordered, ordered):
key=lambda props: idx_for_value.get(props.get("@value"), len(ordered))
)
- def _fix_order(self, jsonld_string, document):
+ def _sort_unordered_lists(self, data, sort=True):
+ """Give every JSON-LD list whose order carries no meaning a canonical
+ order, since rdflib serializes them in an arbitrary order that changes
+ from execution to execution.
+
+ Objects that only differ in the label of a blank node still get an
+ arbitrary order, because rdflib generates those labels randomly.
"""
- Fix order of rdfa tags in jsonld string
- by checking the appearance order in the HTML
+ if isinstance(data, list):
+ for item in data:
+ self._sort_unordered_lists(item)
+ if sort:
+ data.sort(key=partial(json.dumps, sort_keys=True))
+ elif isinstance(data, dict):
+ for key, value in data.items():
+ # The order of the items of an @list is meaningful.
+ self._sort_unordered_lists(value, sort=key != "@list")
+
+ def _fix_order(self, json_objects, document):
+ """
+ Fix order of rdfa tags by checking the appearance order in the HTML
"""
- json_objects = json.loads(jsonld_string)
-
html, head = document.xpath("/html"), document.xpath("//head")
if not html or not head:
return json_objects
@@ -164,9 +180,12 @@ def extract_items(self, document, base_url=None, expanded=True):
if isinstance(jsonld_string, bytes):
jsonld_string = jsonld_string.decode("utf-8")
+ json_objects = json.loads(jsonld_string)
+
+ # hacks to fix the ordering of the output (see issues 116 and 146),
+ # they should be disabled once rdflib and PyRDFa fix themselves
+ self._sort_unordered_lists(json_objects)
try:
- # hack to fix the ordering of multi-value properties (see issue 116)
- # it should be disabled once PyRDFA fixes itself
- return self._fix_order(jsonld_string, document)
+ return self._fix_order(json_objects, document)
except:
- return json.loads(jsonld_string)
+ return json_objects
diff --git a/tests/samples/misc/rdfa_inlist.html b/tests/samples/misc/rdfa_inlist.html
new file mode 100644
index 00000000..5a0bafbc
--- /dev/null
+++ b/tests/samples/misc/rdfa_inlist.html
@@ -0,0 +1,10 @@
+
+
+ Ordered list of creators
+
+
+ Charlie
+ Alice
+ Bob
+
+
diff --git a/tests/test_rdfa.py b/tests/test_rdfa.py
index b3d81a7e..23044856 100644
--- a/tests/test_rdfa.py
+++ b/tests/test_rdfa.py
@@ -122,6 +122,45 @@ def test_wikipedia_xhtml_rdfa_no_prefix(self):
self.assertJsonLDEqual(data, expected)
+ def test_deterministic_order(self):
+ # See https://github.com/scrapinghub/extruct/issues/146
+ rdfae = RDFaExtractor()
+ base_url = "http://www.example.com/index.html"
+
+ data = rdfae.extract(
+ get_testdata("w3crdfa", "w3c.rdfaprimer.example011.html"),
+ base_url=base_url,
+ )
+ self.assertEqual(
+ [obj["@id"] for obj in data],
+ [
+ "http://example.com/bob/photos/sunset.jpg",
+ "http://www.example.com/alice/posts/jos_barbecue",
+ "http://www.example.com/alice/posts/trouble_with_bob",
+ "http://www.example.com/index.html",
+ ],
+ )
+
+ data = rdfae.extract(
+ get_testdata("w3crdfa", "w3c.rdfaprimer.example009.html"),
+ base_url=base_url,
+ )
+ self.assertEqual(
+ [obj["@id"] for obj in data[0]["http://www.w3.org/ns/rdfa#usesVocabulary"]],
+ ["http://creativecommons.org/ns#", "http://purl.org/dc/terms/"],
+ )
+
+ def test_list_order_kept(self):
+ data = RDFaExtractor().extract(
+ get_testdata("misc", "rdfa_inlist.html"),
+ base_url="http://www.example.com/index.html",
+ )
+ creators = data[0]["http://purl.org/dc/terms/creator"][0]["@list"]
+ self.assertEqual(
+ [creator["@value"] for creator in creators],
+ ["Charlie", "Alice", "Bob"],
+ )
+
def test_expanded_opengraph_support(self):
body = get_testdata("misc", "expanded_OG_support_test.html")
expected = json.loads(