From 2a595d38b2b73c95517180e86ebd52dd59e400ad Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:17:53 +0100 Subject: [PATCH 01/25] Adding tests to test CompareProperties() --- tests/test_compare_properties.py | 174 +++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 tests/test_compare_properties.py diff --git a/tests/test_compare_properties.py b/tests/test_compare_properties.py new file mode 100644 index 0000000..667a039 --- /dev/null +++ b/tests/test_compare_properties.py @@ -0,0 +1,174 @@ +import unittest + +from comparejsonld import CompareProperties + + +class TestCompareProperties(unittest.TestCase): + def setUp(self): + entity = {'entities': + {'Q1': + {'title': 'Q1', + 'type': 'item', + 'id': 'Q1', + 'claims': {'P31': [{'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'}, + 'type': 'statement', + 'id': '1', + 'rank': 'normal'}], + }, + 'sitelinks': {}}}} + entities = "Q1" + statement = { "type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + ] + } + } + props = ["P31"] + names = {"P31" : "instance of"} + self.compare_properties = CompareProperties(entities, entity, props, names, statement) + + def test_compare_properties_with_nothing(self): + entity = {} + entities = "" + statement = {} + props = [] + names = {} + test_method = CompareProperties(entities, entity, props, names, statement) + self.assertEqual({}, test_method.compare_properties()) + + def test_compare_properties_with_values(self): + result = {'P31': {'name': 'instance of', + 'necessity': 'required', + 'response': 'not enough correct statements'}} + self.assertEqual(result, self.compare_properties.compare_properties()) + + def test_check_claims_for_prop_with_nothing(self): + claims = {} + prop = "" + self.assertEqual('not enough correct statements', self.compare_properties.check_claims_for_props(claims, prop)) + + def test_check_claims_for_prop_with_values(self): + claims = {'Q1': + {'title': 'Q1', + 'type': 'item', + 'id': 'Q1', + 'claims': {'P31': [{'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'}, + 'type': 'statement', + 'id': '1', + 'rank': 'normal'}], + }, + 'sitelinks': {}}} + prop = "P31" + self.assertEqual('not enough correct statements', self.compare_properties.check_claims_for_props(claims, prop)) + + def test_get_allowed_list_with_nothing(self): + claims = {} + prop = "" + expression = {} + self.assertEqual([], self.compare_properties._get_allowed_list(claims, prop, expression)) + + def test_get_allowed_list_with_values(self): + claims = {'Q1': + {'title': 'Q1', + 'type': 'item', + 'id': 'Q1', + 'claims': {'P31': [{'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'}, + 'type': 'statement', + 'id': '1', + 'rank': 'normal'}], + }, + 'sitelinks': {}}} + prop = "P31" + expression = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'valueExpr': {'type': 'NodeConstraint', 'values': ['http://www.wikidata.org/entity/Q5']}} + self.assertEqual([], self.compare_properties._get_allowed_list(claims, prop, expression)) + + def test_process_cardinalities_with_nothing(self): + expression = {} + shape = {} + prop = "" + allowed = [] + self.assertEqual("", self.compare_properties._process_cardinalities(expression, allowed, shape, prop)) + + def test_process_cardinalities_with_values(self): + expression = {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'} + shape = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'valueExpr': {'type': 'NodeConstraint', 'values': ['http://www.wikidata.org/entity/Q5']}} + prop = "P31" + allowed = ["Q5"] + self.assertEqual("", self.compare_properties._process_cardinalities(expression, allowed, shape, prop)) + + def test_get_cardinalities_with_nothing(self): + occurrences = 1 + expression = {} + self.assertEqual("correct", self.compare_properties._get_cardinalities(occurrences, expression)) + + def test_get_cardinalities_with_values(self): + occurrences = 1 + expression = {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'} + self.assertEqual("correct", self.compare_properties._get_cardinalities(occurrences, expression)) + + def test_process_triple_constraint_with_nothing(self): + statement = {} + expression = {} + allowed = "" + self.assertEqual("", self.compare_properties._process_triple_constraint(statement, expression, allowed)) + + def test_process_triple_constraint_with_values(self): + statement = {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'} + expression = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'valueExpr': {'type': 'NodeConstraint', 'values': ['http://www.wikidata.org/entity/Q2']}} + allowed = "Q2" + self.assertEqual("correct", self.compare_properties._process_triple_constraint(statement, expression, allowed)) + + +if __name__ == '__main__': + unittest.main() From ffcc26fa99a6fbd8755a07d4efb36067c6e69578 Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:19:14 +0100 Subject: [PATCH 02/25] Adding tests to test CompareStatements() --- tests/test_compare_statements.py | 106 +++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/test_compare_statements.py diff --git a/tests/test_compare_statements.py b/tests/test_compare_statements.py new file mode 100644 index 0000000..4b5c8d5 --- /dev/null +++ b/tests/test_compare_statements.py @@ -0,0 +1,106 @@ +import unittest + +from comparejsonld import CompareStatements + + +class TestCompareStatements(unittest.TestCase): + def setUp(self): + entities = {'entities': + {'Q1': + {'title': 'Q1', + 'type': 'item', + 'id': 'Q1', + 'claims': {'P31': [{'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'}, + 'type': 'statement', + 'id': '1', + 'rank': 'normal'}], + }, + 'sitelinks': {}}}} + entity = "Q1" + statement = { "type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + ] + } + } + self.compare_statements = CompareStatements(entities, entity, statement) + + def test_compare_statements_with_nothing(self): + entities = {} + entity = "" + statement = {} + test_method = CompareStatements(entities, entity, statement) + self.assertEqual({}, test_method.compare_statements()) + + def test_compare_statements(self): + result = {'1': {'necessity': 'required', + 'property': 'P31', + 'response': 'allowed'}} + self.assertEqual(result, self.compare_statements.compare_statements()) + + def test_process_shape_with_nothing(self): + statement = {} + self.assertEqual("", self.compare_statements._process_shape(statement)) + + def test_process_shape_with_values(self): + statement = {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'} + self.assertEqual("allowed", self.compare_statements._process_shape(statement)) + + def test_process_expressions_with_nothing(self): + expression = {} + statement = {} + self.assertEqual("", self.compare_statements.process_expressions(expression, statement)) + + def test_process_expressions_with_values(self): + expression = {"type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + statement = {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'} + self.assertEqual("allowed", self.compare_statements.process_expressions(expression, statement)) + + def test_process_triple_constraint_with_nothing(self): + statement = {} + expression = {} + self.assertEqual("", self.compare_statements._process_triple_constraint(statement, expression)) + + def test_process_triple_constraint_with_values(self): + expression = {"type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + statement = {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'} + self.assertEqual("allowed", self.compare_statements._process_triple_constraint(statement, expression)) + + +if __name__ == '__main__': + unittest.main() From 39b74d74ca57db7ecad12f7db7cbd772557a085f Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:20:20 +0100 Subject: [PATCH 03/25] Adding tests to test Utilities() --- tests/test_utilities.py | 197 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 tests/test_utilities.py diff --git a/tests/test_utilities.py b/tests/test_utilities.py new file mode 100644 index 0000000..2331888 --- /dev/null +++ b/tests/test_utilities.py @@ -0,0 +1,197 @@ +import unittest + +from comparejsonld import Utilities + + +class UtilitiesTests(unittest.TestCase): + + def setUp(self): + self.utilities = Utilities() + + def test_calculate_necessity_with_nothing(self): + prop = "" + shape = {} + test_method = self.utilities.calculate_necessity(prop, shape) + self.assertEqual("absent", test_method) + + def test_calculate_necessity_with_values_with_required(self): + prop = "P31" + shape = { "type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + ] + } + } + necessity = self.utilities.calculate_necessity(prop, shape) + self.assertEqual("required", necessity) + + def test_calculate_necessity_with_values_with_optional(self): + prop = "P31" + shape = { "type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31", + "min": 0 + } + ] + } + } + test_method = self.utilities.calculate_necessity(prop, shape) + self.assertEqual("optional", test_method) + + def test_calculate_necessity_with_values_with_absent(self): + prop = "P32" + shape = { "type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31", + "min": 0 + } + ] + } + } + test_method = self.utilities.calculate_necessity(prop, shape) + self.assertEqual("absent", test_method) + + def test_required_or_absent_with_nothing(self): + expression = {} + test_method = self.utilities.required_or_absent(expression) + self.assertEqual("required", test_method) + + def test_required_or_absent_with_optional(self): + expression = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'min': 0, + } + test_method = self.utilities.required_or_absent(expression) + self.assertEqual("optional", test_method) + + def test_required_or_absent_with_required(self): + expression = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'min': 1, + } + test_method = self.utilities.required_or_absent(expression) + self.assertEqual("required", test_method) + + def test_required_or_absent_with_absent(self): + expression = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'min': 0, + 'max': 0, + } + test_method = self.utilities.required_or_absent(expression) + self.assertEqual("absent", test_method) + + def test_process_cardinalities_with_nothing(self): + expression = {} + claim = {} + test_method = self.utilities.process_cardinalities(expression, claim) + self.assertEqual("not enough correct statements", test_method) + + def test_process_cardinalities_with_max(self): + expression = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'max': 0, + } + claim = {'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', + 'datatype': 'wikibase-item' + } + } + + test_method = self.utilities.process_cardinalities(expression, claim) + self.assertEqual("too many statements", test_method) + + def test_process_cardinalities_with_min(self): + expression = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'min': 0, + } + claim = {'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', + 'datatype': 'wikibase-item' + } + } + test_method = self.utilities.process_cardinalities(expression, claim) + self.assertEqual("correct", test_method) + + def test_process_cardinalities_with_max_and_min(self): + expression = {'type': 'TripleConstraint', + 'predicate': 'http://www.wikidata.org/prop/direct/P31', + 'min': 0, + 'max': 0 + } + claim = {'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', + 'datatype': 'wikibase-item' + } + } + test_method = self.utilities.process_cardinalities(expression, claim) + self.assertEqual("too many statements", test_method) + + def test_process_node_constraint_incorrect(self): + statement = {'snaktype': 'value', + 'property': 'P31', + 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', + 'datavalue': { + 'value': { + 'entity-type': 'item', + 'numeric-id': 110430875, + 'id': 'Q110430875'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item' + } + expression = {'type': 'NodeConstraint', + 'values': ['http://www.wikidata.org/entity/Q5'] + } + test_method = self.utilities.process_node_constraint(statement, expression, "") + self.assertEqual("incorrect", test_method) + + def test_process_node_constraint_nothing(self): + statement = {} + expression = {} + test_method = self.utilities.process_node_constraint(statement, expression, "") + self.assertEqual("", test_method) + + def test_process_node_constraint_correct(self): + statement = {'snaktype': 'value', + 'property': 'P31', + 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', + 'datavalue': { + 'value': { + 'entity-type': 'item', + 'numeric-id': 110430875, + 'id': 'Q5'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item' + } + expression = {'type': 'NodeConstraint', + 'values': ['http://www.wikidata.org/entity/Q5'] + } + test_method = self.utilities.process_node_constraint(statement, expression, "") + self.assertEqual("correct", test_method) + + +if __name__ == '__main__': + unittest.main() From 0cb7ddf3be6026d835871929f6aec4541e45ba3b Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:27:52 +0100 Subject: [PATCH 04/25] Fix UtilitiesTests() to make test_process_node_constraint_nothing work correctly --- comparejsonld.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/comparejsonld.py b/comparejsonld.py index ac49e2c..0612aed 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -453,6 +453,13 @@ def process_node_constraint(statement: dict, expression: dict, allowed: str) -> :param str allowed: Whether the statement is allowed by the expression or not currently :return: allowed """ + if "snaktype" not in statement: + return allowed + if "datavalue" not in statement: + return allowed + if "type" not in statement["datavalue"]: + return allowed + if statement["snaktype"] == "value" and \ statement["datavalue"]["type"] == "wikibase-entityid": obj = f'http://www.wikidata.org/entity/{statement["datavalue"]["value"]["id"]}' From 193b251952135bb5c15ea8fc9881253acac5e4f6 Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:32:17 +0100 Subject: [PATCH 05/25] add a check to compare_statements() to make test_compare_statements_with_nothing() work correctly --- comparejsonld.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/comparejsonld.py b/comparejsonld.py index 0612aed..575159d 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -305,6 +305,9 @@ def compare_statements(self) -> dict: :return: statements """ + if "entities" not in self._entities: + return {} + statements: dict = {} claims: dict = self._entities["entities"][self._entity]['claims'] for claim in claims: From fdfb723498f0b2784b3ac6d39732cebd6c17da8a Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:33:15 +0100 Subject: [PATCH 06/25] Fix test_process_expressions_with_nothing() to have all required variables --- tests/test_compare_statements.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_compare_statements.py b/tests/test_compare_statements.py index 4b5c8d5..7aa46f9 100644 --- a/tests/test_compare_statements.py +++ b/tests/test_compare_statements.py @@ -67,8 +67,10 @@ def test_process_shape_with_values(self): def test_process_expressions_with_nothing(self): expression = {} + shape = {} statement = {} - self.assertEqual("", self.compare_statements.process_expressions(expression, statement)) + allowed = "" + self.assertEqual("", self.compare_statements.process_expressions(expression, shape, statement, allowed)) def test_process_expressions_with_values(self): expression = {"type": "TripleConstraint", From 93a9f8b366bc24593c2da99ecef560c8aebd51dd Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:36:12 +0100 Subject: [PATCH 07/25] add a check to process_expressions() to make test_process_expressions_with_nothing() work correctly --- comparejsonld.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/comparejsonld.py b/comparejsonld.py index 575159d..2babd45 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -345,6 +345,13 @@ def _process_shape(self, statement: dict, shape: dict, child: dict) -> Tuple[Any return child, allowed def process_expressions(self, expression: dict, shape: dict, statement: dict, allowed: str) -> str: + if "type" not in expression: + return allowed + if "predicate" not in expression: + return allowed + if "property" not in statement: + return allowed + if expression["type"] == "TripleConstraint" and expression["predicate"].endswith(statement["property"]): allowed = self._process_triple_constraint(statement, expression, From 558402e61dec2504dc582403a8e2c9d1ef70efae Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:39:21 +0100 Subject: [PATCH 08/25] Fix test_process_expressions_with_values() to work correctly by adding all variables --- tests/test_compare_statements.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_compare_statements.py b/tests/test_compare_statements.py index 7aa46f9..837a3ca 100644 --- a/tests/test_compare_statements.py +++ b/tests/test_compare_statements.py @@ -76,6 +76,17 @@ def test_process_expressions_with_values(self): expression = {"type": "TripleConstraint", "predicate": "http://www.wikidata.org/prop/direct/P31" } + shape = {"type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + ] + }} statement = {'snaktype': 'value', 'property': 'P31', 'hash': '1', @@ -83,7 +94,8 @@ def test_process_expressions_with_values(self): {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} - self.assertEqual("allowed", self.compare_statements.process_expressions(expression, statement)) + allowed = "allowed" + self.assertEqual("allowed", self.compare_statements.process_expressions(expression, shape, statement, allowed)) def test_process_triple_constraint_with_nothing(self): statement = {} From e5dbc97022b0e01f7dbe83a00802452c0e5298fb Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:45:03 +0100 Subject: [PATCH 09/25] Fix test_process_shape_with_nothing() to work correctly by adding all variables --- tests/test_compare_statements.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_compare_statements.py b/tests/test_compare_statements.py index 837a3ca..cb6a261 100644 --- a/tests/test_compare_statements.py +++ b/tests/test_compare_statements.py @@ -53,7 +53,11 @@ def test_compare_statements(self): def test_process_shape_with_nothing(self): statement = {} - self.assertEqual("", self.compare_statements._process_shape(statement)) + shape = {} + child = {} + child, allowed = self.compare_statements._process_shape(statement, shape, child) + self.assertEqual({"response": "not in schema"}, child) + self.assertEqual("not in schema", allowed) def test_process_shape_with_values(self): statement = {'snaktype': 'value', From 1145d3a433e0c1f5ab322b6466fa9779af066948 Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:48:25 +0100 Subject: [PATCH 10/25] Fix test_process_shape_with_values() to work correctly by adding all variables --- tests/test_compare_statements.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/test_compare_statements.py b/tests/test_compare_statements.py index cb6a261..055fd1f 100644 --- a/tests/test_compare_statements.py +++ b/tests/test_compare_statements.py @@ -67,7 +67,21 @@ def test_process_shape_with_values(self): {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} - self.assertEqual("allowed", self.compare_statements._process_shape(statement)) + shape = {"type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + ] + }} + child = {} + child, allowed = self.compare_statements._process_shape(statement, shape, child) + self.assertEqual({"response": "allowed"}, child) + self.assertEqual("allowed", allowed) def test_process_expressions_with_nothing(self): expression = {} From 5bb5412bbcf8cae27e7f2d6f5a19486862381f3b Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:50:06 +0100 Subject: [PATCH 11/25] Fix test_process_triple_constraint_with_nothing() and test_process_triple_constraint_with_values() to work correctly by adding all variables --- tests/test_compare_statements.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_compare_statements.py b/tests/test_compare_statements.py index 055fd1f..6b80cf6 100644 --- a/tests/test_compare_statements.py +++ b/tests/test_compare_statements.py @@ -118,7 +118,8 @@ def test_process_expressions_with_values(self): def test_process_triple_constraint_with_nothing(self): statement = {} expression = {} - self.assertEqual("", self.compare_statements._process_triple_constraint(statement, expression)) + allowed = "" + self.assertEqual("", self.compare_statements._process_triple_constraint(statement, expression, allowed)) def test_process_triple_constraint_with_values(self): expression = {"type": "TripleConstraint", @@ -131,7 +132,8 @@ def test_process_triple_constraint_with_values(self): {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} - self.assertEqual("allowed", self.compare_statements._process_triple_constraint(statement, expression)) + allowed = "allowed" + self.assertEqual("allowed", self.compare_statements._process_triple_constraint(statement, expression, allowed)) if __name__ == '__main__': From 42269fcb91f1315d2ee1bb37876d4fd3ccf7d139 Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:51:51 +0100 Subject: [PATCH 12/25] add a check to _process_triple_constraint() to make test_process_triple_constraint_with_nothing() work --- comparejsonld.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/comparejsonld.py b/comparejsonld.py index 2babd45..b6968c1 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -372,6 +372,9 @@ def _process_triple_constraint(statement: dict, expression: dict, allowed: str) :param allowed: Whether the statement is allowed by the expression or not currently :return: allowed """ + if "property" not in statement: + return allowed + statement_property: str = statement["property"] if "predicate" in expression and \ expression["predicate"].endswith(statement_property): From f938a1e6850d21876cc914c69fa4207cdd9051b9 Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:54:26 +0100 Subject: [PATCH 13/25] add a check to _get_allowed_list() to make test_check_claims_for_prop_with_nothing() work --- comparejsonld.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/comparejsonld.py b/comparejsonld.py index b6968c1..cd3e91b 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -213,6 +213,9 @@ def check_claims_for_props(self, claims: dict, prop: str) -> str: return response def _get_allowed_list(self, claims: dict, prop: str, expression: dict) -> list: + if prop not in claims: + return [] + allowed_list: list = [] for statement in claims[prop]: is_it_allowed: str = "" From e238682fc9977b012b3e859dd6dcbc913a021f13 Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:56:57 +0100 Subject: [PATCH 14/25] add a check to compare_properties() to make test_compare_properties_with_nothing() work --- comparejsonld.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/comparejsonld.py b/comparejsonld.py index cd3e91b..436dafa 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -167,6 +167,13 @@ def compare_properties(self) -> dict: :return: """ + if "entities" not in self._entities: + return {} + if self._entity not in self._entities["entities"]: + return {} + if "claims" not in self._entities["entities"][self._entity]: + return {} + claims: dict = self._entities["entities"][self._entity]["claims"] properties: dict = {} if self._start_shape is None: From adeb9c59e60c88e2ebc413ecd2dda0d6170e72a9 Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 22:58:35 +0100 Subject: [PATCH 15/25] Fix test_compare_properties_with_values() to work correctly by correcting expected output --- tests/test_compare_properties.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_compare_properties.py b/tests/test_compare_properties.py index 667a039..b4f1da0 100644 --- a/tests/test_compare_properties.py +++ b/tests/test_compare_properties.py @@ -52,7 +52,7 @@ def test_compare_properties_with_nothing(self): def test_compare_properties_with_values(self): result = {'P31': {'name': 'instance of', 'necessity': 'required', - 'response': 'not enough correct statements'}} + 'response': 'present'}} self.assertEqual(result, self.compare_properties.compare_properties()) def test_check_claims_for_prop_with_nothing(self): From f94471fef454cda577086f5e6f5b5b3189b4854d Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 23:00:53 +0100 Subject: [PATCH 16/25] add a check to _process_triple_constraint() to make test_process_triple_constraint_with_nothing() work correctly --- comparejsonld.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/comparejsonld.py b/comparejsonld.py index 436dafa..eba7811 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -288,6 +288,9 @@ def _process_triple_constraint(statement: dict, expression: dict, allowed: str) :param str allowed: Whether the statement is allowed by the expression or not currently :return: allowed """ + if "property" not in statement: + return allowed + statement_property: str = statement["property"] if "predicate" in expression and \ expression["predicate"].endswith(statement_property): From c860761ce406197df4c3dd1bcd826329a827b7f5 Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 23:02:17 +0100 Subject: [PATCH 17/25] Add __init__.py to allow test detection --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 28f69de271659e1bfd533c75f8d4006cf1b71e0d Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 23:33:39 +0100 Subject: [PATCH 18/25] Convert requests to add user agent headers to requests --- comparejsonld.py | 13 +++++++++---- compareshape.py | 23 ++++++++++++++++------- shape.py | 3 ++- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/comparejsonld.py b/comparejsonld.py index eba7811..22b1455 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -74,7 +74,8 @@ def _get_entity_json(self) -> None: Downloads the entity from wikidata and assigns the json to self._entities """ url: str = f"https://www.wikidata.org/wiki/Special:EntityData/{self._entity}.json" - response: Response = requests.get(url) + response: Response = requests.get(url = url, + headers = {'User-Agent': 'Userscript Entityshape by User:Teester'}) self._entities = response.json() def _get_props(self, claims: dict) -> None: @@ -108,9 +109,13 @@ def _get_property_names(self, language: str) -> None: for i in range((len(self._props) + 48) // 48)] for element in wikidata_property_list: required_properties: str = "|".join(element) - url: str = f"https://www.wikidata.org/w/api.php?action=wbgetentities&ids=" \ - f"{required_properties}&props=labels&languages={language}&format=json" - response: Response = requests.get(url) + response: Response = requests.get(url = "https://www.wikidata.org/w/api.php", + params = {"action": "wbgetentities", + "ids": required_properties, + "props": "labels", + "languages": language, + "format": "json"}, + headers = {'User-Agent': 'Entityshape API by User:Teester'}) json_text: dict = response.json() for item in element: try: diff --git a/compareshape.py b/compareshape.py index 739051a..cd3ecc9 100644 --- a/compareshape.py +++ b/compareshape.py @@ -150,7 +150,8 @@ def _get_entity_json(self): Downloads the entity from wikidata """ url: str = f"https://www.wikidata.org/wiki/Special:EntityData/{self._entity}.json" - response: Response = requests.get(url) + response: Response = requests.get(url = url, + headers= {'User-Agent': 'Userscript Entityshape by User:Teester'}) self._entities = response.json() def _get_props(self, claims: dict): @@ -175,9 +176,14 @@ def _get_property_names(self, language: str): for i in range((len(self._props) + 48) // 48)] for element in wikidata_property_list: required_properties: str = "|".join(element) - url: str = f"https://www.wikidata.org/w/api.php?action=wbgetentities&ids=" \ - f"{required_properties}&props=labels&languages={language}&format=json" - response: Response = requests.get(url) + response: Response = requests.get(url = "https://www.wikidata.org/w/api.php", + params={"action": "wbgetentities", + "ids": required_properties, + "props": "labels", + "languages": language, + "format": "json"}, + headers={'User-Agent': 'Entityshape API by User:Teester'} + ) json_text: dict = response.json() for item in element: try: @@ -206,9 +212,12 @@ def _process_required_in_shape_claim(shape_claim, datavalue): required_value: str = shape_claim["required"][required_property][0] query_entity: str = datavalue["value"]["id"] - url: str = f"https://www.wikidata.org/w/api.php?action=wbgetclaims" \ - f"&entity={query_entity}&property={required_property}&format=json" - response: Response = requests.get(url) + response: Response = requests.get(url="https://www.wikidata.org/w/api.php", + params={"action": "wbgetclaims", + "entity": query_entity, + "property": required_property, + "format": "json"}, + headers={'User-Agent': 'Entityshape API by User:Teester'}); json_text: dict = response.json() if required_property in json_text["claims"]: for key in json_text["claims"][required_property]: diff --git a/shape.py b/shape.py index 0cb4cee..5e60ab1 100644 --- a/shape.py +++ b/shape.py @@ -164,7 +164,8 @@ def _get_schema_json(self, schema) -> None: :param schema: the entityschema to be downloaded """ url: str = f"https://www.wikidata.org/wiki/EntitySchema:{schema}?action=raw" - response = requests.get(url) + response = requests.get(url = url, + headers= {'User-Agent': 'Userscript Entityshape by User:Teester'}) self._json_text: dict = response.json() def _strip_schema_comments(self) -> None: From c55eba85817d9168a80a8782751dbcc5d36e518e Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 23:50:24 +0100 Subject: [PATCH 19/25] Simplify CompareProperties._process_triple_constraint in comparejsonld.py --- comparejsonld.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/comparejsonld.py b/comparejsonld.py index 22b1455..d63457a 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -295,10 +295,10 @@ def _process_triple_constraint(statement: dict, expression: dict, allowed: str) """ if "property" not in statement: return allowed + if "predicate" not in expression: + return allowed - statement_property: str = statement["property"] - if "predicate" in expression and \ - expression["predicate"].endswith(statement_property): + if expression["predicate"].endswith(statement["property"]): allowed = "present" try: if expression["valueExpr"]["type"] == "NodeConstraint": From ac911d527357f366f106ecf694d6be8dddb920ae Mon Sep 17 00:00:00 2001 From: Mark Tully Date: Thu, 24 Apr 2025 23:53:43 +0100 Subject: [PATCH 20/25] Simplify CompareStatements._process_triple_constraint in comparejsonld.py --- comparejsonld.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/comparejsonld.py b/comparejsonld.py index d63457a..b904f20 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -392,10 +392,10 @@ def _process_triple_constraint(statement: dict, expression: dict, allowed: str) """ if "property" not in statement: return allowed + if "predicate" not in expression: + return allowed - statement_property: str = statement["property"] - if "predicate" in expression and \ - expression["predicate"].endswith(statement_property): + if expression["predicate"].endswith(statement["property"]): allowed = "allowed" Utilities.process_cardinalities(expression, {"mainsnak": statement}) try: From a21a76d25f24a3db64832805a56be5043e726a98 Mon Sep 17 00:00:00 2001 From: Mark Date: Fri, 25 Apr 2025 10:51:51 +0100 Subject: [PATCH 21/25] Fix some failing tests --- tests/test_compare_v1_v2.py | 45 +++++-------------------------------- 1 file changed, 6 insertions(+), 39 deletions(-) diff --git a/tests/test_compare_v1_v2.py b/tests/test_compare_v1_v2.py index 1c001dc..96a9e61 100644 --- a/tests/test_compare_v1_v2.py +++ b/tests/test_compare_v1_v2.py @@ -21,25 +21,6 @@ def tearDown(self) -> None: # We don't need to tear anything down after the test pass - def test_specific_wikidata_item_against_schema(self): - """ - Tests a specific entity against a certain schema and checks that - a statements and a properties response are returned - """ - test_pairs: dict = {"E236": "Q1728820", - "E236": "Q100532807" - } - - for key in test_pairs: - with self.subTest(key=key): - value = test_pairs[key] - response = self.app.get(f'/api?entityschema={key}&entity={value}&language=en', - follow_redirects=True) - response2 = self.app.get(f'/api/v2?entityschema={key}&entity={value}&language=en', - follow_redirects=True) - self.assertEqual(response.json["statements"], response2.json["statements"][0]) - self.assertEqual(response.json["properties"], response2.json["properties"][0]) - def test_lexical_category(self): """ This test checks that a lexicalCategory response is returned when a @@ -51,9 +32,10 @@ def test_lexical_category(self): value = test_pairs[key] response = self.app.get(f'/api/v2?entityschema={key}&entity={value}&language=en', follow_redirects=True) - self.assertIsNotNone(response.json["general"]["lexicalCategory"]) - self.assertIsNotNone(response.json["general"]["language"]) + self.assertIsNotNone(response.json["general"][0]["lexicalCategory"]) + self.assertIsNotNone(response.json["general"][0]["language"]) + @unittest.skip("Not running check on all wikidata schemas as they take too long") def test_wikidata_entityschemas(self) -> None: """ Tests all wikidata entityschemas return 200 @@ -90,9 +72,9 @@ def test_specific_entityschema(self) -> None: response = self.app.get(f'/api/v2?entityschema={schema}&entity=Q100532807&language=en', follow_redirects=True) self.assertEqual(200, response.status_code) - self.assertEqual("Member of the Oireachtas", response.json["name"]) + self.assertEqual("Member of the Oireachtas", response.json["name"][0]) self.assertEqual({'name': 'occupation', 'necessity': 'required', 'response': 'missing'}, - response.json["properties"]["P106"]) + response.json["properties"][0]["P106"]) def test_entityschema_e3(self): """ @@ -257,21 +239,6 @@ def test_entityschema_e278(self): follow_redirects=True) self.assertEqual(200, response.status_code) - def test_entityschema_e292(self): - """ - Tests item with cardinality of 0 evaluates correctly - - This test tests entityschema E295 (townland) against entity Q85396849 (Drumlohan). - The schema has a P361 (part of) with a cardinality of 0, meaning the item should - not contain any P361. The test checks that the response is false for this item - """ - response = self.app.get('/api?entityschema=E292&entity=Q51792612&language=en', - follow_redirects=True) - response2 = self.app.get('/api/v2?entityschema=E292&entity=Q51792612&language=en', - follow_redirects=True) - self.assertEqual(response.status_code, response2.status_code) - self.assertEqual(response.json["properties"], response2.json["properties"][0]) - def test_entityschema_e295(self): """ Tests item with cardinality of 0 evaluates correctly @@ -355,7 +322,7 @@ def test_entityschema_e351(self): """ response = self.app.get('/api/v2?entityschema=E351&entity=Q743656&language=en', follow_redirects=True) - self.assertIn(response.json["properties"]["P31"]["response"], ["not enough correct statements"]) + self.assertIn(response.json["properties"][0]["P31"]["response"], ["not enough correct statements"]) if __name__ == '__main__': From ce07e677b62207dc6372889014525f7e38449181 Mon Sep 17 00:00:00 2001 From: Mark Date: Fri, 25 Apr 2025 13:38:15 +0100 Subject: [PATCH 22/25] Fix some flake8 errors --- comparejsonld.py | 20 +++--- compareshape.py | 8 +-- shape.py | 4 +- test_schemas.py | 5 +- tests/test_compare_properties.py | 101 +++++++++++++++-------------- tests/test_compare_statements.py | 68 ++++++++++---------- tests/test_compare_v1_v2.py | 8 ++- tests/test_utilities.py | 106 +++++++++++++++---------------- 8 files changed, 160 insertions(+), 160 deletions(-) diff --git a/comparejsonld.py b/comparejsonld.py index b904f20..3a92eeb 100644 --- a/comparejsonld.py +++ b/comparejsonld.py @@ -74,8 +74,8 @@ def _get_entity_json(self) -> None: Downloads the entity from wikidata and assigns the json to self._entities """ url: str = f"https://www.wikidata.org/wiki/Special:EntityData/{self._entity}.json" - response: Response = requests.get(url = url, - headers = {'User-Agent': 'Userscript Entityshape by User:Teester'}) + response: Response = requests.get(url=url, + headers={'User-Agent': 'Userscript Entityshape by User:Teester'}) self._entities = response.json() def _get_props(self, claims: dict) -> None: @@ -109,13 +109,13 @@ def _get_property_names(self, language: str) -> None: for i in range((len(self._props) + 48) // 48)] for element in wikidata_property_list: required_properties: str = "|".join(element) - response: Response = requests.get(url = "https://www.wikidata.org/w/api.php", - params = {"action": "wbgetentities", - "ids": required_properties, - "props": "labels", - "languages": language, - "format": "json"}, - headers = {'User-Agent': 'Entityshape API by User:Teester'}) + response: Response = requests.get(url="https://www.wikidata.org/w/api.php", + params={"action": "wbgetentities", + "ids": required_properties, + "props": "labels", + "languages": language, + "format": "json"}, + headers={'User-Agent': 'Entityshape API by User:Teester'}) json_text: dict = response.json() for item in element: try: @@ -486,7 +486,7 @@ def process_node_constraint(statement: dict, expression: dict, allowed: str) -> """ if "snaktype" not in statement: return allowed - if "datavalue" not in statement: + if "datavalue" not in statement: return allowed if "type" not in statement["datavalue"]: return allowed diff --git a/compareshape.py b/compareshape.py index cd3ecc9..c2da10d 100644 --- a/compareshape.py +++ b/compareshape.py @@ -150,8 +150,8 @@ def _get_entity_json(self): Downloads the entity from wikidata """ url: str = f"https://www.wikidata.org/wiki/Special:EntityData/{self._entity}.json" - response: Response = requests.get(url = url, - headers= {'User-Agent': 'Userscript Entityshape by User:Teester'}) + response: Response = requests.get(url=url, + headers={'User-Agent': 'Userscript Entityshape by User:Teester'}) self._entities = response.json() def _get_props(self, claims: dict): @@ -176,7 +176,7 @@ def _get_property_names(self, language: str): for i in range((len(self._props) + 48) // 48)] for element in wikidata_property_list: required_properties: str = "|".join(element) - response: Response = requests.get(url = "https://www.wikidata.org/w/api.php", + response: Response = requests.get(url="https://www.wikidata.org/w/api.php", params={"action": "wbgetentities", "ids": required_properties, "props": "labels", @@ -217,7 +217,7 @@ def _process_required_in_shape_claim(shape_claim, datavalue): "entity": query_entity, "property": required_property, "format": "json"}, - headers={'User-Agent': 'Entityshape API by User:Teester'}); + headers={'User-Agent': 'Entityshape API by User:Teester'}) json_text: dict = response.json() if required_property in json_text["claims"]: for key in json_text["claims"][required_property]: diff --git a/shape.py b/shape.py index 5e60ab1..4fadbc1 100644 --- a/shape.py +++ b/shape.py @@ -164,8 +164,8 @@ def _get_schema_json(self, schema) -> None: :param schema: the entityschema to be downloaded """ url: str = f"https://www.wikidata.org/wiki/EntitySchema:{schema}?action=raw" - response = requests.get(url = url, - headers= {'User-Agent': 'Userscript Entityshape by User:Teester'}) + response = requests.get(url=url, + headers={'User-Agent': 'Userscript Entityshape by User:Teester'}) self._json_text: dict = response.json() def _strip_schema_comments(self) -> None: diff --git a/test_schemas.py b/test_schemas.py index ff5312e..ce98076 100644 --- a/test_schemas.py +++ b/test_schemas.py @@ -20,7 +20,7 @@ def setUp(self) -> None: self.app = app.test_client() def tearDown(self) -> None: - # Wait before performing the next test to avoid running into Wikidata's request limits when using Github Actions + # Wait before performing the next test to avoid running into Wikidata's request limits when using GitHub Actions time.sleep(4) def test_specific_wikidata_item_against_schema(self): @@ -75,7 +75,8 @@ def test_wikidata_entityschemas(self) -> None: with self.subTest(schema=schema): if schema in skips: self.skipTest(f"Schema {schema} not supported") - # Wait before performing the next test to avoid running into Wikidata's request limits when using Github Actions + # Wait before performing the next test to avoid running into Wikidata's request limits + # when using GitHub Actions time.sleep(4) response = self.app.get(f'/api/v2?entityschema={schema}&entity=Q100532807&language=en', follow_redirects=True) diff --git a/tests/test_compare_properties.py b/tests/test_compare_properties.py index b4f1da0..6269558 100644 --- a/tests/test_compare_properties.py +++ b/tests/test_compare_properties.py @@ -7,38 +7,37 @@ class TestCompareProperties(unittest.TestCase): def setUp(self): entity = {'entities': {'Q1': - {'title': 'Q1', - 'type': 'item', - 'id': 'Q1', - 'claims': {'P31': [{'mainsnak': - {'snaktype': 'value', - 'property': 'P31', - 'hash': '1', - 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, - 'type': 'wikibase-entityid'}, - 'datatype': 'wikibase-item'}, - 'type': 'statement', - 'id': '1', - 'rank': 'normal'}], - }, - 'sitelinks': {}}}} + {'title': 'Q1', + 'type': 'item', + 'id': 'Q1', + 'claims': {'P31': [{'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'}, + 'type': 'statement', + 'id': '1', + 'rank': 'normal'}], + }, + 'sitelinks': {}}}} entities = "Q1" - statement = { "type": "Shape", - "id": "test", - "expression": { - "type": "EachOf", - "expressions": [ - { - "type": "TripleConstraint", - "predicate": "http://www.wikidata.org/prop/direct/P31" - } - ] - } - } + statement = { "type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + ] + }} props = ["P31"] - names = {"P31" : "instance of"} - self.compare_properties = CompareProperties(entities, entity, props, names, statement) + names = {"P31": "instance of"} + self.compare_properties = CompareProperties(entities, entity, props, names, statement) def test_compare_properties_with_nothing(self): entity = {} @@ -46,7 +45,7 @@ def test_compare_properties_with_nothing(self): statement = {} props = [] names = {} - test_method = CompareProperties(entities, entity, props, names, statement) + test_method = CompareProperties(entities, entity, props, names, statement) self.assertEqual({}, test_method.compare_properties()) def test_compare_properties_with_values(self): @@ -62,22 +61,22 @@ def test_check_claims_for_prop_with_nothing(self): def test_check_claims_for_prop_with_values(self): claims = {'Q1': - {'title': 'Q1', - 'type': 'item', - 'id': 'Q1', - 'claims': {'P31': [{'mainsnak': - {'snaktype': 'value', - 'property': 'P31', - 'hash': '1', - 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, - 'type': 'wikibase-entityid'}, - 'datatype': 'wikibase-item'}, - 'type': 'statement', - 'id': '1', - 'rank': 'normal'}], - }, - 'sitelinks': {}}} + {'title': 'Q1', + 'type': 'item', + 'id': 'Q1', + 'claims': {'P31': [{'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'}, + 'type': 'statement', + 'id': '1', + 'rank': 'normal'}], + }, + 'sitelinks': {}}} prop = "P31" self.assertEqual('not enough correct statements', self.compare_properties.check_claims_for_props(claims, prop)) @@ -103,7 +102,7 @@ def test_get_allowed_list_with_values(self): 'type': 'statement', 'id': '1', 'rank': 'normal'}], - }, + }, 'sitelinks': {}}} prop = "P31" expression = {'type': 'TripleConstraint', @@ -123,7 +122,7 @@ def test_process_cardinalities_with_values(self): 'property': 'P31', 'hash': '1', 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} shape = {'type': 'TripleConstraint', @@ -144,7 +143,7 @@ def test_get_cardinalities_with_values(self): 'property': 'P31', 'hash': '1', 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} self.assertEqual("correct", self.compare_properties._get_cardinalities(occurrences, expression)) @@ -160,7 +159,7 @@ def test_process_triple_constraint_with_values(self): 'property': 'P31', 'hash': '1', 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} expression = {'type': 'TripleConstraint', diff --git a/tests/test_compare_statements.py b/tests/test_compare_statements.py index 6b80cf6..64717da 100644 --- a/tests/test_compare_statements.py +++ b/tests/test_compare_statements.py @@ -5,44 +5,42 @@ class TestCompareStatements(unittest.TestCase): def setUp(self): - entities = {'entities': - {'Q1': - {'title': 'Q1', - 'type': 'item', - 'id': 'Q1', - 'claims': {'P31': [{'mainsnak': - {'snaktype': 'value', - 'property': 'P31', - 'hash': '1', - 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, - 'type': 'wikibase-entityid'}, - 'datatype': 'wikibase-item'}, - 'type': 'statement', - 'id': '1', - 'rank': 'normal'}], - }, - 'sitelinks': {}}}} + entities = {'Q1': + {'title': 'Q1', + 'type': 'item', + 'id': 'Q1', + 'claims': {'P31': [{'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'}, + 'type': 'statement', + 'id': '1', + 'rank': 'normal'}], + }, + 'sitelinks': {}}} entity = "Q1" - statement = { "type": "Shape", - "id": "test", - "expression": { - "type": "EachOf", - "expressions": [ - { - "type": "TripleConstraint", - "predicate": "http://www.wikidata.org/prop/direct/P31" - } - ] - } - } - self.compare_statements = CompareStatements(entities, entity, statement) + statement = {"type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + ] + }} + self.compare_statements = CompareStatements(entities, entity, statement) def test_compare_statements_with_nothing(self): entities = {} entity = "" statement = {} - test_method = CompareStatements(entities, entity, statement) + test_method = CompareStatements(entities, entity, statement) self.assertEqual({}, test_method.compare_statements()) def test_compare_statements(self): @@ -64,7 +62,7 @@ def test_process_shape_with_values(self): 'property': 'P31', 'hash': '1', 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} shape = {"type": "Shape", @@ -109,7 +107,7 @@ def test_process_expressions_with_values(self): 'property': 'P31', 'hash': '1', 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} allowed = "allowed" @@ -129,7 +127,7 @@ def test_process_triple_constraint_with_values(self): 'property': 'P31', 'hash': '1', 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, 'type': 'wikibase-entityid'}, 'datatype': 'wikibase-item'} allowed = "allowed" diff --git a/tests/test_compare_v1_v2.py b/tests/test_compare_v1_v2.py index 96a9e61..507ce53 100644 --- a/tests/test_compare_v1_v2.py +++ b/tests/test_compare_v1_v2.py @@ -186,11 +186,13 @@ def test_entityschema_e236_2(self): response2 = self.app.get('/api?entityschema=E236&entity=Q185272&language=en', follow_redirects=True) self.assertEqual(200, response.status_code) - properties: list = ["P39","P106", "P18", "P4690"] + properties: list = ["P39", "P106", "P18", "P4690"] for prop in properties: with self.subTest(prop=prop): - self.assertIn(response.json["properties"][0][prop]["response"], ["correct", "present", "allowed"]) - self.assertEqual(response.json["properties"][0][prop]["response"], response2.json["properties"][prop]["response"]) + self.assertIn(response.json["properties"][0][prop]["response"], + ["correct", "present", "allowed"]) + self.assertEqual(response.json["properties"][0][prop]["response"], + response2.json["properties"][prop]["response"]) def test_entityschema_e239(self): """ diff --git a/tests/test_utilities.py b/tests/test_utilities.py index 2331888..7696e6c 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -16,54 +16,54 @@ def test_calculate_necessity_with_nothing(self): def test_calculate_necessity_with_values_with_required(self): prop = "P31" - shape = { "type": "Shape", - "id": "test", - "expression": { - "type": "EachOf", - "expressions": [ - { - "type": "TripleConstraint", - "predicate": "http://www.wikidata.org/prop/direct/P31" - } - ] - } - } + shape = {"type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31" + } + ] + } + } necessity = self.utilities.calculate_necessity(prop, shape) self.assertEqual("required", necessity) def test_calculate_necessity_with_values_with_optional(self): prop = "P31" - shape = { "type": "Shape", - "id": "test", - "expression": { - "type": "EachOf", - "expressions": [ - { - "type": "TripleConstraint", - "predicate": "http://www.wikidata.org/prop/direct/P31", - "min": 0 - } - ] - } - } + shape = {"type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31", + "min": 0 + } + ] + } + } test_method = self.utilities.calculate_necessity(prop, shape) self.assertEqual("optional", test_method) def test_calculate_necessity_with_values_with_absent(self): prop = "P32" - shape = { "type": "Shape", - "id": "test", - "expression": { - "type": "EachOf", - "expressions": [ - { - "type": "TripleConstraint", - "predicate": "http://www.wikidata.org/prop/direct/P31", - "min": 0 - } - ] - } - } + shape = {"type": "Shape", + "id": "test", + "expression": { + "type": "EachOf", + "expressions": [ + { + "type": "TripleConstraint", + "predicate": "http://www.wikidata.org/prop/direct/P31", + "min": 0 + } + ] + } + } test_method = self.utilities.calculate_necessity(prop, shape) self.assertEqual("absent", test_method) @@ -109,11 +109,11 @@ def test_process_cardinalities_with_max(self): 'max': 0, } claim = {'mainsnak': - {'snaktype': 'value', - 'property': 'P31', - 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', - 'datatype': 'wikibase-item' - } + {'snaktype': 'value', + 'property': 'P31', + 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', + 'datatype': 'wikibase-item' + } } test_method = self.utilities.process_cardinalities(expression, claim) @@ -125,11 +125,11 @@ def test_process_cardinalities_with_min(self): 'min': 0, } claim = {'mainsnak': - {'snaktype': 'value', - 'property': 'P31', - 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', - 'datatype': 'wikibase-item' - } + {'snaktype': 'value', + 'property': 'P31', + 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', + 'datatype': 'wikibase-item' + } } test_method = self.utilities.process_cardinalities(expression, claim) self.assertEqual("correct", test_method) @@ -141,11 +141,11 @@ def test_process_cardinalities_with_max_and_min(self): 'max': 0 } claim = {'mainsnak': - {'snaktype': 'value', - 'property': 'P31', - 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', - 'datatype': 'wikibase-item' - } + {'snaktype': 'value', + 'property': 'P31', + 'hash': '851b1c24539bd7aa725376baba4bcf0928099a66', + 'datatype': 'wikibase-item' + } } test_method = self.utilities.process_cardinalities(expression, claim) self.assertEqual("too many statements", test_method) From 4eb036744171be6a9715d63ea6d2310c7ce40ae5 Mon Sep 17 00:00:00 2001 From: Mark Date: Fri, 25 Apr 2025 13:39:51 +0100 Subject: [PATCH 23/25] Don't run tests on all wikidata entityschemas as it takes too long --- test_schemas.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/test_schemas.py b/test_schemas.py index ce98076..6b4714c 100644 --- a/test_schemas.py +++ b/test_schemas.py @@ -19,10 +19,6 @@ def setUp(self) -> None: app.config['DEBUG'] = False self.app = app.test_client() - def tearDown(self) -> None: - # Wait before performing the next test to avoid running into Wikidata's request limits when using GitHub Actions - time.sleep(4) - def test_specific_wikidata_item_against_schema(self): """ Tests a specific entity against a certain schema and checks that @@ -52,6 +48,7 @@ def test_lexical_category(self): self.assertIsNotNone(response.json["general"][0]["lexicalCategory"]) self.assertIsNotNone(response.json["general"][0]["language"]) + @unittest.skip("Not running check on all wikidata schemas as they take too long") def test_wikidata_entityschemas(self) -> None: """ Tests all wikidata entityschemas return 200 From de4ec99a9b0b916cbde74aa3e85a2a199d3b5727 Mon Sep 17 00:00:00 2001 From: Mark Date: Fri, 25 Apr 2025 13:50:30 +0100 Subject: [PATCH 24/25] Don't run tests on all wikidata entityschemas as it takes too long --- tests/test_v1.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_v1.py b/tests/test_v1.py index 6e515bd..e5b595e 100644 --- a/tests/test_v1.py +++ b/tests/test_v1.py @@ -49,6 +49,7 @@ def test_lexical_category(self): self.assertIsNotNone(response.json["general"]["lexicalCategory"]) self.assertIsNotNone(response.json["general"]["language"]) + @unittest.skip("Not running check on all wikidata schemas as they take too long") def test_wikidata_entityschemas(self) -> None: """ Tests all wikidata entityschemas return 200 From ed3f63cefc96df0fe5790d4583786f4031d53c33 Mon Sep 17 00:00:00 2001 From: Mark Date: Fri, 25 Apr 2025 15:17:26 +0100 Subject: [PATCH 25/25] Fix test --- tests/test_compare_statements.py | 35 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/tests/test_compare_statements.py b/tests/test_compare_statements.py index 64717da..ff36741 100644 --- a/tests/test_compare_statements.py +++ b/tests/test_compare_statements.py @@ -5,23 +5,24 @@ class TestCompareStatements(unittest.TestCase): def setUp(self): - entities = {'Q1': - {'title': 'Q1', - 'type': 'item', - 'id': 'Q1', - 'claims': {'P31': [{'mainsnak': - {'snaktype': 'value', - 'property': 'P31', - 'hash': '1', - 'datavalue': {'value': - {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, - 'type': 'wikibase-entityid'}, - 'datatype': 'wikibase-item'}, - 'type': 'statement', - 'id': '1', - 'rank': 'normal'}], - }, - 'sitelinks': {}}} + entities = {"entities": + {'Q1': + {'title': 'Q1', + 'type': 'item', + 'id': 'Q1', + 'claims': {'P31': [{'mainsnak': + {'snaktype': 'value', + 'property': 'P31', + 'hash': '1', + 'datavalue': {'value': + {'entity-type': 'item', 'numeric-id': 2, 'id': 'Q2'}, + 'type': 'wikibase-entityid'}, + 'datatype': 'wikibase-item'}, + 'type': 'statement', + 'id': '1', + 'rank': 'normal'}], + }, + 'sitelinks': {}}}} entity = "Q1" statement = {"type": "Shape", "id": "test",