Enforce deterministic order in RDFA

README.rst

@@ -143,9 +143,7 @@ First fetch the HTML using python-requests and then feed the response body to ``
                                        'https://www.optimizesmart.com/wp-content/uploads/2010/07/open-graph-protocol.jpg'),
                                      ( 'og:image:secure_url',
                                        'https://www.optimizesmart.com/wp-content/uploads/2010/07/open-graph-protocol.jpg')]}],
-    'rdfa': [ { '@id': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/#header',
-                'http://www.w3.org/1999/xhtml/vocab#role': [ { '@id': 'http://www.w3.org/1999/xhtml/vocab#banner'}]},
-              { '@id': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/',
+    'rdfa': [ { '@id': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/',
                 'article:modified_time': [ { '@value': '2018-03-09T16:26:35+00:00'}],
                 'article:published_time': [ { '@value': '2010-07-02T18:57:23+00:00'}],
                 'article:publisher': [ { '@value': 'https://www.facebook.com/optimizesmart/'}],
@@ -171,7 +169,9 @@ First fetch the HTML using python-requests and then feed the response body to ``
                 'http://ogp.me/ns#type': [{'@value': 'article'}],
                 'http://ogp.me/ns#updated_time': [ { '@value': '2018-03-09T16:26:35+00:00'}],
                 'http://ogp.me/ns#url': [ { '@value': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/'}],
-                'https://api.w.org/': [ { '@id': 'https://www.optimizesmart.com/wp-json/'}]}]}
+                'https://api.w.org/': [ { '@id': 'https://www.optimizesmart.com/wp-json/'}]},
+              { '@id': 'https://www.optimizesmart.com/how-to-use-open-graph-protocol/#header',
+                'http://www.w3.org/1999/xhtml/vocab#role': [ { '@id': 'http://www.w3.org/1999/xhtml/vocab#banner'}]}]}
 
 Select syntaxes
 +++++++++++++++

extruct/rdfa.py

@@ -8,6 +8,7 @@
 import logging
 import re
 from collections import defaultdict
+from functools import partial
 
 rdflib_logger = logging.getLogger("rdflib")
 rdflib_logger.setLevel(logging.ERROR)
@@ -110,13 +111,28 @@ def _sort(self, unordered, ordered):
             key=lambda props: idx_for_value.get(props.get("@value"), len(ordered))
         )
 
-    def _fix_order(self, jsonld_string, document):
+    def _sort_unordered_lists(self, data, sort=True):
+        """Give every JSON-LD list whose order carries no meaning a canonical
+        order, since rdflib serializes them in an arbitrary order that changes
+        from execution to execution.
+
+        Objects that only differ in the label of a blank node still get an
+        arbitrary order, because rdflib generates those labels randomly.
         """
-        Fix order of rdfa tags in jsonld string
-        by checking the appearance order in the HTML
+        if isinstance(data, list):
+            for item in data:
+                self._sort_unordered_lists(item)
+            if sort:
+                data.sort(key=partial(json.dumps, sort_keys=True))
+        elif isinstance(data, dict):
+            for key, value in data.items():
+                # The order of the items of an @list is meaningful.
+                self._sort_unordered_lists(value, sort=key != "@list")
+
+    def _fix_order(self, json_objects, document):
+        """
+        Fix order of rdfa tags by checking the appearance order in the HTML
         """
-        json_objects = json.loads(jsonld_string)
-
         html, head = document.xpath("/html"), document.xpath("//head")
         if not html or not head:
             return json_objects
@@ -164,9 +180,12 @@ def extract_items(self, document, base_url=None, expanded=True):
         if isinstance(jsonld_string, bytes):
             jsonld_string = jsonld_string.decode("utf-8")
 
+        json_objects = json.loads(jsonld_string)
+
+        # hacks to fix the ordering of the output (see issues 116 and 146),
+        # they should be disabled once rdflib and PyRDFa fix themselves
+        self._sort_unordered_lists(json_objects)
         try:
-            # hack to fix the ordering of multi-value properties (see issue 116)
-            # it should be disabled once PyRDFA fixes itself
-            return self._fix_order(jsonld_string, document)
+            return self._fix_order(json_objects, document)
         except:
-            return json.loads(jsonld_string)
+            return json_objects

tests/samples/misc/rdfa_inlist.html

@@ -0,0 +1,10 @@
+<html>
+  <head>
+    <title>Ordered list of creators</title>
+  </head>
+  <body vocab="http://purl.org/dc/terms/" resource="http://www.example.com/index.html">
+    <span property="creator" inlist>Charlie</span>
+    <span property="creator" inlist>Alice</span>
+    <span property="creator" inlist>Bob</span>
+  </body>
+</html>

tests/test_rdfa.py

@@ -122,6 +122,45 @@ def test_wikipedia_xhtml_rdfa_no_prefix(self):
 
         self.assertJsonLDEqual(data, expected)
 
+    def test_deterministic_order(self):
+        # See https://github.com/scrapinghub/extruct/issues/146
+        rdfae = RDFaExtractor()
+        base_url = "http://www.example.com/index.html"
+
+        data = rdfae.extract(
+            get_testdata("w3crdfa", "w3c.rdfaprimer.example011.html"),
+            base_url=base_url,
+        )
+        self.assertEqual(
+            [obj["@id"] for obj in data],
+            [
+                "http://example.com/bob/photos/sunset.jpg",
+                "http://www.example.com/alice/posts/jos_barbecue",
+                "http://www.example.com/alice/posts/trouble_with_bob",
+                "http://www.example.com/index.html",
+            ],
+        )
+
+        data = rdfae.extract(
+            get_testdata("w3crdfa", "w3c.rdfaprimer.example009.html"),
+            base_url=base_url,
+        )
+        self.assertEqual(
+            [obj["@id"] for obj in data[0]["http://www.w3.org/ns/rdfa#usesVocabulary"]],
+            ["http://creativecommons.org/ns#", "http://purl.org/dc/terms/"],
+        )
+
+    def test_list_order_kept(self):
+        data = RDFaExtractor().extract(
+            get_testdata("misc", "rdfa_inlist.html"),
+            base_url="http://www.example.com/index.html",
+        )
+        creators = data[0]["http://purl.org/dc/terms/creator"][0]["@list"]
+        self.assertEqual(
+            [creator["@value"] for creator in creators],
+            ["Charlie", "Alice", "Bob"],
+        )
+
     def test_expanded_opengraph_support(self):
         body = get_testdata("misc", "expanded_OG_support_test.html")
         expected = json.loads(