diff --git a/site/cds_rdm/inspire_harvester/reader.py b/site/cds_rdm/inspire_harvester/reader.py index e5f884d1..faf59c75 100644 --- a/site/cds_rdm/inspire_harvester/reader.py +++ b/site/cds_rdm/inspire_harvester/reader.py @@ -64,9 +64,30 @@ def _get_json(self, url, headers): current_app.logger.debug("Request response is successful (200).") return response.json() + def _merge_documents_from_ui(self, inspire_record): + """Copy ``documents`` from the UI JSON when expanded JSON omitted them. + + ``expanded+json`` keeps machine CDS identifiers the matcher needs, but + often drops ``documents``. ``ui+json`` has the files the website shows. + """ + record_id = str(inspire_record["id"]) + base_url = current_app.config["CDS_HARVESTER_INSPIRE_API_URL"].rstrip("/") + data = self._get_json( + f"{base_url}/{record_id}", + {"Accept": "application/vnd+inspire.record.ui+json"}, + ) + documents = [ + doc + for doc in (data.get("metadata") or {}).get("documents") or [] + if not doc.get("hidden") + ] + if documents: + inspire_record.setdefault("metadata", {})["documents"] = documents + def _iter(self, url, *args, **kwargs): """Yields HTTP response.""" - # header set to include additional data (external file URLs and more detailed metadata + # Prefer expanded+json for structured ids (e.g. CDS schema/value). File + # lists missing there are filled from ui+json in _merge_documents_from_ui. headers = {"Accept": "application/vnd+inspire.record.expanded+json"} # Three nested loops: # - outer: re-harvest from the start when we got fewer records than INSPIRE @@ -105,6 +126,9 @@ def _iter(self, url, *args, **kwargs): continue seen_ids.add(record_id) new_in_pass += 1 + metadata = inspire_record.get("metadata") + if metadata is not None and not metadata.get("documents"): + self._merge_documents_from_ui(inspire_record) current_app.logger.debug( f"Sending INSPIRE record #{record_id} to transformer." ) diff --git a/site/tests/inspire_harvester/test_reader.py b/site/tests/inspire_harvester/test_reader.py index fe9dd662..4377ea03 100644 --- a/site/tests/inspire_harvester/test_reader.py +++ b/site/tests/inspire_harvester/test_reader.py @@ -211,3 +211,90 @@ def side_effect(url, headers=None): ] assert mock_get.call_count == MAX_HARVEST_PASSES assert "after max retries; stopping" in caplog.text + + +def test_reader_merges_documents_from_ui_json(running_app): + """Backfill documents from ui+json when expanded+json omitted them.""" + search_page = { + "hits": { + "total": 2, + "hits": [ + { + "id": "2170337", + "metadata": { + "control_number": 2170337, + "external_system_identifiers": [ + {"schema": "CDS", "value": "2973593"} + ], + }, + }, + { + "id": "3218251", + "metadata": { + "control_number": 3218251, + "documents": [ + { + "key": "already-there", + "filename": "from-expanded.pdf", + "url": "https://inspirehep.net/files/already-there", + } + ], + }, + }, + ], + }, + "links": {}, + } + ui_detail = { + "metadata": { + "documents": [ + { + "key": "e4e6db10201f1d059989950429fdcca9", + "filename": "document", + "url": ( + "https://inspirehep.net/files/" + "e4e6db10201f1d059989950429fdcca9" + ), + "fulltext": True, + "hidden": False, + }, + { + "key": "hidden-xml", + "filename": "secret.xml", + "url": "https://inspirehep.net/files/hidden-xml", + "fulltext": True, + "hidden": True, + }, + ] + } + } + + def side_effect(url, headers=None): + mock_response = Mock() + mock_response.status_code = 200 + accept = (headers or {}).get("Accept", "") + if url.rstrip("/").endswith("/2170337") and "ui+json" in accept: + mock_response.json.return_value = ui_detail + else: + mock_response.json.return_value = search_page + return mock_response + + with patch("requests.get", side_effect=side_effect) as mock_get: + reader = InspireHTTPReader(inspire_id="2170337") + records = list(reader.read()) + + by_id = {str(r["id"]): r for r in records} + assert by_id["2170337"]["metadata"]["documents"] == [ + ui_detail["metadata"]["documents"][0] + ] + assert by_id["2170337"]["metadata"]["external_system_identifiers"] == [ + {"schema": "CDS", "value": "2973593"} + ] + assert by_id["3218251"]["metadata"]["documents"][0]["key"] == "already-there" + ui_calls = [ + call + for call in mock_get.call_args_list + if (call.kwargs.get("headers") or {}).get("Accept", "").endswith("ui+json") + ] + assert len(ui_calls) == 1 + assert ui_calls[0].args[0].endswith("/2170337")