diff --git a/site/cds_rdm/inspire_harvester/load/files.py b/site/cds_rdm/inspire_harvester/load/files.py index 701f868b..c2fc4c2b 100644 --- a/site/cds_rdm/inspire_harvester/load/files.py +++ b/site/cds_rdm/inspire_harvester/load/files.py @@ -19,6 +19,9 @@ from invenio_vocabularies.datastreams.errors import WriterError from cds_rdm.inspire_harvester.load.draft import DraftLifecycleManager +from cds_rdm.inspire_harvester.transform.mappers.config import ( + INSPIRE_CORRECTION_MATERIALS, +) @dataclass @@ -180,6 +183,43 @@ def sync(self, draft, record, incoming_record, logger, import_files=True): file_content = self.fetch(inspire_url, logger) self._upload_file(draft, file, file_content, logger) logger.info(f"{len(new_files)} files successfully created.") + self._set_default_preview(draft, logger) + + def _set_default_preview(self, draft, logger): + """Set the draft's default preview from the files it now holds. + + Correction files (erratum/corrigendum/addendum, tagged in their + description by the files mapper) are never the preview. Among the + rest a PDF is preferred, else the first file. If every file is a + correction the field is left unset. + """ + service = current_rdm_records_service + data = service.read_draft(self.identity, draft.id).to_dict() + entries = data["files"].get("entries", {}) + + candidates = [ + file_data + for file_data in entries.values() + if not any( + term in ((file_data.get("metadata") or {}).get("description") or "").lower() + for term in INSPIRE_CORRECTION_MATERIALS + ) + ] + if not candidates: + return + + def _is_pdf(file_data): + mimetype = (file_data.get("mimetype") or "").lower() + return mimetype == "application/pdf" or file_data["key"].lower().endswith( + ".pdf" + ) + + key = next((f for f in candidates if _is_pdf(f)), candidates[0])["key"] + if data["files"].get("default_preview") == key: + return + data["files"]["default_preview"] = key + service.update_draft(self.identity, draft.id, data) + logger.debug(f"Default preview set. | details: key={key}") def _upload_file(self, draft, file_data, file_content, logger): """Initialize, upload, and commit a single file to the draft.""" diff --git a/site/cds_rdm/inspire_harvester/transform/context.py b/site/cds_rdm/inspire_harvester/transform/context.py index 96cad121..d7ecbd67 100644 --- a/site/cds_rdm/inspire_harvester/transform/context.py +++ b/site/cds_rdm/inspire_harvester/transform/context.py @@ -8,7 +8,7 @@ """INSPIRE to CDS harvester context module.""" from dataclasses import dataclass, field -from typing import List, Optional +from typing import List, Optional, Tuple from cds_rdm.inspire_harvester.transform.resource_types import ResourceType @@ -23,3 +23,5 @@ class MetadataSerializationContext: errors: List[str] = field(default_factory=list) # Extra DOIs (non-main) to append as related identifiers. extra_related_dois: List[str] = field(default_factory=list) + # (DOI, material) pairs, mapped as related identifiers (issupplementedby). + correction_dois: List[Tuple[str, str]] = field(default_factory=list) diff --git a/site/cds_rdm/inspire_harvester/transform/mappers/article.py b/site/cds_rdm/inspire_harvester/transform/mappers/article.py index 94a4e24c..a347dfc5 100644 --- a/site/cds_rdm/inspire_harvester/transform/mappers/article.py +++ b/site/cds_rdm/inspire_harvester/transform/mappers/article.py @@ -12,7 +12,10 @@ from flask import current_app from cds_rdm.inspire_harvester.transform.mappers.basic_metadata import TitleMapper -from cds_rdm.inspire_harvester.transform.mappers.files import FilesMapper +from cds_rdm.inspire_harvester.transform.mappers.files import ( + FilesMapper, + get_correction_material, +) from cds_rdm.inspire_harvester.transform.mappers.identifiers import DOIMapper from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase @@ -25,10 +28,15 @@ def filter(self, file): """Filters files based on given criteria.""" material = file.get("material") source = file.get("source") - if not (source or material): + + if source == "arxiv": return False - if (not material or material == "publication") and source != "arxiv": - return True + + is_publication = material == "publication" + is_correction = bool(get_correction_material(file, file.get("filename", ""))) + is_unqualified = bool(source) and not material + + return is_publication or is_correction or is_unqualified @dataclass(frozen=True) @@ -79,4 +87,4 @@ def map_value(self, src_record, ctx, logger): source = abstract.get("source", "").lower() if source and source not in ["arxiv", "cds"]: return abstract["value"] - return abstracts[0]["value"] \ No newline at end of file + return abstracts[0]["value"] diff --git a/site/cds_rdm/inspire_harvester/transform/mappers/config.py b/site/cds_rdm/inspire_harvester/transform/mappers/config.py new file mode 100644 index 00000000..0508ad40 --- /dev/null +++ b/site/cds_rdm/inspire_harvester/transform/mappers/config.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2026 CERN. +# +# CDS-RDM is free software; you can redistribute it and/or modify it under +# the terms of the MIT License; see LICENSE file for more details. + +"""INSPIRE to CDS harvester configuration.""" + +# INSPIRE ``material`` values that mark a DOI/document as a correction of +# the main publication. Their DOIs are mapped to related identifiers +# (``issupplementedby``) and their files are tagged with the material. +INSPIRE_CORRECTION_MATERIALS = { + "erratum": {"resource_type": "publication-other"}, + "corrigendum": {"resource_type": "publication-other"}, + "addendum": {"resource_type": "publication-other"}, +} diff --git a/site/cds_rdm/inspire_harvester/transform/mappers/files.py b/site/cds_rdm/inspire_harvester/transform/mappers/files.py index b9e250a9..dc7dc97a 100644 --- a/site/cds_rdm/inspire_harvester/transform/mappers/files.py +++ b/site/cds_rdm/inspire_harvester/transform/mappers/files.py @@ -9,9 +9,25 @@ from dataclasses import dataclass +from cds_rdm.inspire_harvester.transform.mappers.config import ( + INSPIRE_CORRECTION_MATERIALS, +) from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase +def get_correction_material(item, filename=""): + """Return the correction material of a DOI or document, if any. + + INSPIRE sets ``material`` on some documents, but often only the filename + (e.g. ``erratum.pdf``) tells it apart. + """ + material = (item.get("material") or "").lower() + if material in INSPIRE_CORRECTION_MATERIALS: + return material + name = filename.lower() + return next((q for q in INSPIRE_CORRECTION_MATERIALS if q in name), None) + + @dataclass(frozen=True) class FilesMapper(MapperBase): """Mapper for files.""" @@ -59,6 +75,7 @@ def map_value(self, src_record, ctx, logger): } rdm_files_entries[filename] = file_details + correction = get_correction_material(file, filename) logger.info(f"File mapped: {file_details}. File name: {filename}.") file_metadata = {"source": source, "source_url": file["url"]} @@ -69,6 +86,10 @@ def map_value(self, src_record, ctx, logger): if file_original_url: file_metadata["original_url"] = file_original_url + if correction: + # Displayed title + file_metadata["description"] = correction + if file_metadata: rdm_files_entries[filename]["metadata"] = file_metadata diff --git a/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py b/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py index 5f7cd999..91632d93 100644 --- a/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py +++ b/site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py @@ -16,6 +16,9 @@ from idutils.validators import is_doi, is_url, is_urn from cds_rdm import schemes +from cds_rdm.inspire_harvester.transform.mappers.config import ( + INSPIRE_CORRECTION_MATERIALS, +) from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase @@ -117,9 +120,19 @@ def map_value(self, src_record, ctx, logger): seen = set() unique_dois = [] for d in dois: - if d["value"] not in seen: - unique_dois.append(d) - seen.add(d["value"]) + if d["value"] in seen: + continue + seen.add(d["value"]) + correction = (d.get("material") or "").lower() + if correction in INSPIRE_CORRECTION_MATERIALS: + if is_doi(d["value"]): + ctx.correction_dois.append((d["value"], correction)) + else: + ctx.errors.append( + f"DOI validation failed. | details: doi={d['value']}" + ) + continue + unique_dois.append(d) for doi in reversed(unique_dois): if not self.filter(doi): @@ -132,9 +145,7 @@ def map_value(self, src_record, ctx, logger): d for d in unique_dois if d.get("value", "").startswith(DATACITE_PREFIX) ] other_dois = [ - d - for d in unique_dois - if not d.get("value", "").startswith(DATACITE_PREFIX) + d for d in unique_dois if not d.get("value", "").startswith(DATACITE_PREFIX) ] if len(cds_dois) > 1: @@ -374,6 +385,20 @@ def map_value(self, src_record, ctx, logger): } ) + for doi, correction in ctx.correction_dois: + identifiers.append( + { + "identifier": doi, + "scheme": "doi", + "relation_type": {"id": "issupplementedby"}, + "resource_type": { + "id": INSPIRE_CORRECTION_MATERIALS[correction][ + "resource_type" + ] + }, + } + ) + identifiers.append( { "scheme": "inspire", diff --git a/site/tests/inspire_harvester/test_transformer.py b/site/tests/inspire_harvester/test_transformer.py index f47c7a8c..f40b9fa8 100644 --- a/site/tests/inspire_harvester/test_transformer.py +++ b/site/tests/inspire_harvester/test_transformer.py @@ -8,10 +8,12 @@ """INSPIRE harvester transformer tests.""" from unittest.mock import Mock, patch +import pytest from edtf.parser.grammar import ParseException from cds_rdm.inspire_harvester.logger import Logger from cds_rdm.inspire_harvester.transform.context import MetadataSerializationContext +from cds_rdm.inspire_harvester.transform.mappers.article import ArticleFilesMapper from cds_rdm.inspire_harvester.transform.mappers.basic_metadata import ( AdditionalDescriptionsMapper, AdditionalTitlesMapper, @@ -1540,3 +1542,123 @@ def test_transform_conference_mapper_fetch_fails(running_app): } ] assert ctx.errors == [] + + +def _ctx(): + return MetadataSerializationContext( + resource_type=ResourceType.ARTICLE, inspire_id="12345" + ) + + +@pytest.mark.parametrize("material", ["erratum", "corrigendum", "addendum", "Erratum"]) +@patch("cds_rdm.inspire_harvester.transform.mappers.identifiers.is_doi") +def test_transform_dois_correction_is_not_main(mock_is_doi, material, running_app): + """Erratum/corrigendum/addendum DOIs never become the main DOI.""" + mock_is_doi.return_value = True + src_record = { + "metadata": { + "dois": [ + {"value": "10.1000/correction", "material": material}, + {"value": "10.1000/main", "material": "publication"}, + ] + } + } + ctx = _ctx() + + result = DOIMapper().map_value(src_record, ctx, Logger(inspire_id="12345")) + + assert result["doi"]["identifier"] == "10.1000/main" + assert ctx.correction_dois == [("10.1000/correction", material.lower())] + assert ctx.extra_related_dois == [] + assert not ctx.errors + + +@patch("cds_rdm.inspire_harvester.transform.mappers.identifiers.is_doi") +def test_transform_dois_correction_invalid(mock_is_doi, running_app): + """An invalid correction DOI is reported and skipped.""" + mock_is_doi.side_effect = lambda v: v != "bad" + src_record = { + "metadata": { + "dois": [ + {"value": "10.1000/main"}, + {"value": "bad", "material": "erratum"}, + ] + } + } + ctx = _ctx() + + DOIMapper().map_value(src_record, ctx, Logger(inspire_id="12345")) + + assert ctx.correction_dois == [] + assert any("DOI validation failed" in e for e in ctx.errors) + + +@patch("cds_rdm.inspire_harvester.transform.mappers.identifiers.is_doi") +def test_transform_related_identifiers_correction_dois(mock_is_doi, running_app): + """Correction DOIs become issupplementedby related identifiers.""" + mock_is_doi.return_value = True + ctx = _ctx() + ctx.correction_dois.append(("10.1000/err", "erratum")) + ctx.correction_dois.append(("10.1000/add", "addendum")) + src_record = {"metadata": {"dois": [{"value": "10.1000/main"}]}} + + result = RelatedIdentifiersMapper().map_value( + src_record, ctx, Logger(inspire_id="12345") + ) + + for doi in ("10.1000/err", "10.1000/add"): + assert { + "identifier": doi, + "scheme": "doi", + "relation_type": {"id": "issupplementedby"}, + "resource_type": {"id": "publication-other"}, + } in result + + +def _documents(*filenames, **extra): + return { + "metadata": { + "documents": [ + { + "filename": name, + "key": f"key{i}", + "url": f"https://example.com/{i}", + **extra, + } + for i, name in enumerate(filenames) + ] + } + } + + +def test_transform_files_tags_correction_files(running_app): + """Correction files are tagged in their description; the filename is unchanged.""" + src_record = _documents("Erratum.pdf", "document.pdf") + + result = FilesMapper().map_value(src_record, _ctx(), Logger(inspire_id="12345")) + + assert "default_preview" not in result + assert result["entries"]["Erratum.pdf"]["metadata"]["description"] == "erratum" + assert "description" not in result["entries"]["document.pdf"]["metadata"] + + +def test_transform_files_tags_correction_from_material(running_app): + """The correction is detected from material as well as from the filename.""" + src_record = _documents("a.pdf", "b.pdf") + src_record["metadata"]["documents"][0]["material"] = "Corrigendum" + + result = FilesMapper().map_value(src_record, _ctx(), Logger(inspire_id="12345")) + + assert result["entries"]["a.pdf"]["metadata"]["description"] == "corrigendum" + assert "description" not in result["entries"]["b.pdf"]["metadata"] + + +def test_article_files_mapper_keeps_correction_files(running_app): + """Article files without source/material are dropped, corrections are kept.""" + src_record = _documents("document.pdf", "erratum.pdf") + + result = ArticleFilesMapper().map_value( + src_record, _ctx(), Logger(inspire_id="12345") + ) + + assert list(result["entries"]) == ["erratum.pdf"] diff --git a/site/tests/inspire_harvester/test_writer.py b/site/tests/inspire_harvester/test_writer.py index e790dab3..ef7d7690 100644 --- a/site/tests/inspire_harvester/test_writer.py +++ b/site/tests/inspire_harvester/test_writer.py @@ -18,6 +18,7 @@ from invenio_vocabularies.datastreams import StreamEntry from invenio_vocabularies.datastreams.errors import WriterError +from cds_rdm.inspire_harvester.load import files as files_module from cds_rdm.inspire_harvester.load.files import FileSynchronizer from cds_rdm.inspire_harvester.load.matcher import MatchResult from cds_rdm.inspire_harvester.writer import InspireWriter @@ -868,3 +869,89 @@ def test_writer_1_existing_found_new_version_creation_failed( created_records.to_dict()["hits"]["hits"][0]["versions"]["index"] == 1 _cleanup_record(created_records.to_dict()["hits"]["hits"][0]["id"]) + + +@pytest.fixture() +def mock_service(): + """Replace the RDM service proxy in the files module (no app context needed).""" + service = Mock() + original = files_module.current_rdm_records_service + files_module.current_rdm_records_service = service + yield service + files_module.current_rdm_records_service = original + + +def _draft_data(entries, default_preview=None): + return { + "files": { + "enabled": True, + "default_preview": default_preview, + "entries": { + key: {"key": key, "metadata": metadata} + for key, metadata in entries.items() + }, + } + } + + +def _set_preview(mock_service, entries, default_preview=None): + mock_service.read_draft.return_value.to_dict.return_value = _draft_data( + entries, default_preview + ) + FileSynchronizer()._set_default_preview(Mock(id="draft-id"), Mock()) + + +def test_set_default_preview_skips_corrections(mock_service): + """A correction file is never the preview, whatever the order.""" + _set_preview( + mock_service, + {"erratum.pdf": {"description": "erratum"}, "document.pdf": {}}, + ) + + data = mock_service.update_draft.call_args[0][2] + assert data["files"]["default_preview"] == "document.pdf" + + +def test_set_default_preview_prefers_pdf(mock_service): + """A PDF is preferred over other files, with the first file as fallback.""" + _set_preview(mock_service, {"plot.png": {}, "document.pdf": {}}) + assert mock_service.update_draft.call_args[0][2]["files"][ + "default_preview" + ] == "document.pdf" + + _set_preview(mock_service, {"plot.png": {}, "data.csv": {}}) + assert mock_service.update_draft.call_args[0][2]["files"][ + "default_preview" + ] == "plot.png" + + +def test_set_default_preview_description_is_case_insensitive(mock_service): + """Description matching ignores case and works on substrings.""" + _set_preview( + mock_service, + {"a.pdf": {"description": "Addendum to the paper"}, "b.pdf": {}}, + ) + + assert mock_service.update_draft.call_args[0][2]["files"][ + "default_preview" + ] == "b.pdf" + + +def test_set_default_preview_noop_cases(mock_service): + """No update when every file is a correction or the preview is already set.""" + _set_preview( + mock_service, + {"erratum.pdf": {"description": "erratum"}, "add.pdf": {"description": "addendum"}}, + ) + _set_preview(mock_service, {"document.pdf": {}}, default_preview="document.pdf") + + mock_service.update_draft.assert_not_called() + + +def test_set_default_preview_file_without_metadata(mock_service): + """A file whose metadata is None (not just missing) can still be previewed.""" + _set_preview(mock_service, {"document.pdf": None, "erratum.pdf": {"description": "erratum"}}) + + assert mock_service.update_draft.call_args[0][2]["files"][ + "default_preview" + ] == "document.pdf"