Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 25 additions & 1 deletion site/cds_rdm/inspire_harvester/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,30 @@ def _get_json(self, url, headers):
current_app.logger.debug("Request response is successful (200).")
return response.json()

def _merge_documents_from_ui(self, inspire_record):

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

aren't we waiting for Micha's response regarding this? shouldn't it be in draft? I am not sure we will need this...

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes I opened this pr before Micha's message and forgot to close it. Closed now.

"""Copy ``documents`` from the UI JSON when expanded JSON omitted them.

``expanded+json`` keeps machine CDS identifiers the matcher needs, but
often drops ``documents``. ``ui+json`` has the files the website shows.
"""
record_id = str(inspire_record["id"])
base_url = current_app.config["CDS_HARVESTER_INSPIRE_API_URL"].rstrip("/")
data = self._get_json(
f"{base_url}/{record_id}",
{"Accept": "application/vnd+inspire.record.ui+json"},
)
documents = [
doc
for doc in (data.get("metadata") or {}).get("documents") or []
if not doc.get("hidden")
]
if documents:
inspire_record.setdefault("metadata", {})["documents"] = documents

def _iter(self, url, *args, **kwargs):
"""Yields HTTP response."""
# header set to include additional data (external file URLs and more detailed metadata
# Prefer expanded+json for structured ids (e.g. CDS schema/value). File
# lists missing there are filled from ui+json in _merge_documents_from_ui.
headers = {"Accept": "application/vnd+inspire.record.expanded+json"}
# Three nested loops:
# - outer: re-harvest from the start when we got fewer records than INSPIRE
Expand Down Expand Up @@ -105,6 +126,9 @@ def _iter(self, url, *args, **kwargs):
continue
seen_ids.add(record_id)
new_in_pass += 1
metadata = inspire_record.get("metadata")
if metadata is not None and not metadata.get("documents"):
self._merge_documents_from_ui(inspire_record)
current_app.logger.debug(
f"Sending INSPIRE record #{record_id} to transformer."
)
Expand Down
87 changes: 87 additions & 0 deletions site/tests/inspire_harvester/test_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,3 +211,90 @@ def side_effect(url, headers=None):
]
assert mock_get.call_count == MAX_HARVEST_PASSES
assert "after max retries; stopping" in caplog.text


def test_reader_merges_documents_from_ui_json(running_app):
"""Backfill documents from ui+json when expanded+json omitted them."""
search_page = {
"hits": {
"total": 2,
"hits": [
{
"id": "2170337",
"metadata": {
"control_number": 2170337,
"external_system_identifiers": [
{"schema": "CDS", "value": "2973593"}
],
},
},
{
"id": "3218251",
"metadata": {
"control_number": 3218251,
"documents": [
{
"key": "already-there",
"filename": "from-expanded.pdf",
"url": "https://inspirehep.net/files/already-there",
}
],
},
},
],
},
"links": {},
}
ui_detail = {
"metadata": {
"documents": [
{
"key": "e4e6db10201f1d059989950429fdcca9",
"filename": "document",
"url": (
"https://inspirehep.net/files/"
"e4e6db10201f1d059989950429fdcca9"
),
"fulltext": True,
"hidden": False,
},
{
"key": "hidden-xml",
"filename": "secret.xml",
"url": "https://inspirehep.net/files/hidden-xml",
"fulltext": True,
"hidden": True,
},
]
}
}

def side_effect(url, headers=None):
mock_response = Mock()
mock_response.status_code = 200
accept = (headers or {}).get("Accept", "")
if url.rstrip("/").endswith("/2170337") and "ui+json" in accept:
mock_response.json.return_value = ui_detail
else:
mock_response.json.return_value = search_page
return mock_response

with patch("requests.get", side_effect=side_effect) as mock_get:
reader = InspireHTTPReader(inspire_id="2170337")
records = list(reader.read())

by_id = {str(r["id"]): r for r in records}
assert by_id["2170337"]["metadata"]["documents"] == [
ui_detail["metadata"]["documents"][0]
]
assert by_id["2170337"]["metadata"]["external_system_identifiers"] == [
{"schema": "CDS", "value": "2973593"}
]
assert by_id["3218251"]["metadata"]["documents"][0]["key"] == "already-there"
ui_calls = [
call
for call in mock_get.call_args_list
if (call.kwargs.get("headers") or {}).get("Accept", "").endswith("ui+json")
]
assert len(ui_calls) == 1
assert ui_calls[0].args[0].endswith("/2170337")
Loading