Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions site/cds_rdm/inspire_harvester/load/matcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,16 @@ def match_migrated(self, entry, logger):
for cds_id in retrieve_identifiers(identifiers, "cds")
if legacy_cds_pattern.match(str(cds_id))
]
# Also use the CDS id taken from INSPIRE external_system_identifiers
# during transform (_inspire_ctx). Needed when metadata.identifiers
# never got the cds scheme (those records were wrongly created).
ctx_cds_id = (entry.get("_inspire_ctx") or {}).get("cds_id")

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't CDS id already be in identifiers (already transformed record, in CDS-RDM format), retrieved in legacy_ids variable?
I suggest investigating why it is not already there, since transformer should retrieve it, instead of adding extra context check. the line above is already specialised in getting the CDS id.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah after looking more into it, those cds ids werent on the inspire records at harvest time (external_system_identifiers), which is why they got created. the next day inspire was updated and the cds ids showed up there, so if we harvested them again now the mapper would put them on metadata.identifiers and theyd be skipped as still on old cds.
image

if (
ctx_cds_id
and legacy_cds_pattern.match(str(ctx_cds_id))
and str(ctx_cds_id) not in {str(i) for i in legacy_ids}
):
legacy_ids.append(str(ctx_cds_id))
matched_ids = []
unresolved = []
for cds_id in legacy_ids:
Expand Down
41 changes: 41 additions & 0 deletions site/tests/inspire_harvester/test_matcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,47 @@ def test_matcher_skips_fallback_search_when_legacy_pidstore_lookup_misses(
mock_get_legacy.assert_called_once_with("2633876")


@patch("cds_rdm.inspire_harvester.load.matcher.RecordMatcher._get_legacy_cds")
@patch("cds_rdm.inspire_harvester.load.matcher.current_rdm_records_service.search")
@patch("cds_rdm.inspire_harvester.load.matcher.get_pid_by_legacy_recid")
def test_matcher_skips_using_inspire_ctx_cds_id_when_identifiers_missing(
mock_get_pid, mock_search, mock_get_legacy, running_app
):
"""Legacy CDS id in _inspire_ctx alone is enough to skip (not create)."""
matcher = RecordMatcher()
logger = Mock()
stream_entry = StreamEntry(
{
"id": "2170337",
"metadata": {
"title": "Test",
"resource_type": {"id": "publication-conferencepaper"},
"identifiers": [],
"related_identifiers": [
{
"scheme": "inspire",
"identifier": "2170337",
"relation_type": {"id": "isvariantformof"},
}
],
},
"files": {"enabled": False},
"parent": {"access": {"owned_by": {"user": 2}}},
"access": {"record": "public", "files": "public"},
"_inspire_ctx": {"cds_id": "2973593", "versions": []},
}
)
mock_get_pid.side_effect = NoResultFound()
mock_get_legacy.return_value = Mock(status_code=200, headers={})

result = matcher.match(stream_entry, inspire_id="2170337", logger=logger)

assert result.unmigrated is True
assert result.matched_ids == ["2973593"]
mock_search.assert_not_called()
mock_get_legacy.assert_called_once_with("2973593")


@patch("cds_rdm.inspire_harvester.load.matcher.current_rdm_records_service.read_latest")
@patch("cds_rdm.inspire_harvester.load.matcher.get_pid_by_legacy_recid")
def test_matcher_finds_record_by_legacy_recid(
Expand Down
Loading