Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions site/cds_rdm/inspire_harvester/load/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
from invenio_vocabularies.datastreams.errors import WriterError

from cds_rdm.inspire_harvester.load.draft import DraftLifecycleManager
from cds_rdm.inspire_harvester.transform.mappers.config import (
INSPIRE_CORRECTION_MATERIALS,
)


@dataclass
Expand Down Expand Up @@ -180,6 +183,43 @@ def sync(self, draft, record, incoming_record, logger, import_files=True):
file_content = self.fetch(inspire_url, logger)
self._upload_file(draft, file, file_content, logger)
logger.info(f"{len(new_files)} files successfully created.")
self._set_default_preview(draft, logger)

def _set_default_preview(self, draft, logger):
"""Set the draft's default preview from the files it now holds.

Correction files (erratum/corrigendum/addendum, tagged in their
description by the files mapper) are never the preview. Among the
rest a PDF is preferred, else the first file. If every file is a
correction the field is left unset.
"""
service = current_rdm_records_service
data = service.read_draft(self.identity, draft.id).to_dict()
entries = data["files"].get("entries", {})

candidates = [
file_data
for file_data in entries.values()
if not any(
term in ((file_data.get("metadata") or {}).get("description") or "").lower()
for term in INSPIRE_CORRECTION_MATERIALS
)
]
if not candidates:
return

def _is_pdf(file_data):
mimetype = (file_data.get("mimetype") or "").lower()
return mimetype == "application/pdf" or file_data["key"].lower().endswith(
".pdf"
)

key = next((f for f in candidates if _is_pdf(f)), candidates[0])["key"]
if data["files"].get("default_preview") == key:
return
data["files"]["default_preview"] = key
service.update_draft(self.identity, draft.id, data)
logger.debug(f"Default preview set. | details: key={key}")

def _upload_file(self, draft, file_data, file_content, logger):
"""Initialize, upload, and commit a single file to the draft."""
Expand Down
4 changes: 3 additions & 1 deletion site/cds_rdm/inspire_harvester/transform/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"""INSPIRE to CDS harvester context module."""

from dataclasses import dataclass, field
from typing import List, Optional
from typing import List, Optional, Tuple

from cds_rdm.inspire_harvester.transform.resource_types import ResourceType

Expand All @@ -23,3 +23,5 @@ class MetadataSerializationContext:
errors: List[str] = field(default_factory=list)
# Extra DOIs (non-main) to append as related identifiers.
extra_related_dois: List[str] = field(default_factory=list)
# (DOI, material) pairs, mapped as related identifiers (issupplementedby).
correction_dois: List[Tuple[str, str]] = field(default_factory=list)
18 changes: 13 additions & 5 deletions site/cds_rdm/inspire_harvester/transform/mappers/article.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@
from flask import current_app

from cds_rdm.inspire_harvester.transform.mappers.basic_metadata import TitleMapper
from cds_rdm.inspire_harvester.transform.mappers.files import FilesMapper
from cds_rdm.inspire_harvester.transform.mappers.files import (
FilesMapper,
get_correction_material,
)
from cds_rdm.inspire_harvester.transform.mappers.identifiers import DOIMapper
from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase

Expand All @@ -25,10 +28,15 @@ def filter(self, file):
"""Filters files based on given criteria."""
material = file.get("material")
source = file.get("source")
if not (source or material):

if source == "arxiv":
return False
if (not material or material == "publication") and source != "arxiv":
return True

is_publication = material == "publication"
is_correction = bool(get_correction_material(file, file.get("filename", "")))
is_unqualified = bool(source) and not material

return is_publication or is_correction or is_unqualified


@dataclass(frozen=True)
Expand Down Expand Up @@ -79,4 +87,4 @@ def map_value(self, src_record, ctx, logger):
source = abstract.get("source", "").lower()
if source and source not in ["arxiv", "cds"]:
return abstract["value"]
return abstracts[0]["value"]
return abstracts[0]["value"]
17 changes: 17 additions & 0 deletions site/cds_rdm/inspire_harvester/transform/mappers/config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# -*- coding: utf-8 -*-
#
# Copyright (C) 2026 CERN.
#
# CDS-RDM is free software; you can redistribute it and/or modify it under
# the terms of the MIT License; see LICENSE file for more details.

"""INSPIRE to CDS harvester configuration."""

# INSPIRE ``material`` values that mark a DOI/document as a correction of
# the main publication. Their DOIs are mapped to related identifiers
# (``issupplementedby``) and their files are tagged with the material.
INSPIRE_CORRECTION_MATERIALS = {
"erratum": {"resource_type": "publication-other"},
"corrigendum": {"resource_type": "publication-other"},
"addendum": {"resource_type": "publication-other"},
Comment on lines +15 to +16

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these 2 are not handled yet in the migrator-kit. Are we sure they come from INSPIRE? If so, maybe we can already add them to the list we have in the migrator-kit.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They are on INSPIRE https://github.com/inspirehep/inspire-schemas/blob/23c49f243fd9fe1e10557a5d92f22965d88b1246/inspire_schemas/records/elements/material.yml#L4

For the migrator-kit: addendum is mentioned here, but not in related identifiers.

Also, the decision log only has erratum, but the issue mentioned all three, so I'm not sure

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you confirm with the library and ammend the decision log and migrator kit if agreed please?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

}
21 changes: 21 additions & 0 deletions site/cds_rdm/inspire_harvester/transform/mappers/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,25 @@

from dataclasses import dataclass

from cds_rdm.inspire_harvester.transform.mappers.config import (
INSPIRE_CORRECTION_MATERIALS,
)
from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase


def get_correction_material(item, filename=""):
"""Return the correction material of a DOI or document, if any.

INSPIRE sets ``material`` on some documents, but often only the filename
(e.g. ``erratum.pdf``) tells it apart.
"""
material = (item.get("material") or "").lower()
if material in INSPIRE_CORRECTION_MATERIALS:
return material
name = filename.lower()
return next((q for q in INSPIRE_CORRECTION_MATERIALS if q in name), None)


@dataclass(frozen=True)
class FilesMapper(MapperBase):
"""Mapper for files."""
Expand Down Expand Up @@ -59,6 +75,7 @@ def map_value(self, src_record, ctx, logger):
}

rdm_files_entries[filename] = file_details
correction = get_correction_material(file, filename)
logger.info(f"File mapped: {file_details}. File name: {filename}.")

file_metadata = {"source": source, "source_url": file["url"]}
Expand All @@ -69,6 +86,10 @@ def map_value(self, src_record, ctx, logger):
if file_original_url:
file_metadata["original_url"] = file_original_url

if correction:
# Displayed title
file_metadata["description"] = correction

if file_metadata:
rdm_files_entries[filename]["metadata"] = file_metadata

Expand Down
37 changes: 31 additions & 6 deletions site/cds_rdm/inspire_harvester/transform/mappers/identifiers.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@
from idutils.validators import is_doi, is_url, is_urn

from cds_rdm import schemes
from cds_rdm.inspire_harvester.transform.mappers.config import (
INSPIRE_CORRECTION_MATERIALS,
)
from cds_rdm.inspire_harvester.transform.mappers.mapper import MapperBase


Expand Down Expand Up @@ -117,9 +120,19 @@ def map_value(self, src_record, ctx, logger):
seen = set()
unique_dois = []
for d in dois:
if d["value"] not in seen:
unique_dois.append(d)
seen.add(d["value"])
if d["value"] in seen:
continue
seen.add(d["value"])
correction = (d.get("material") or "").lower()
if correction in INSPIRE_CORRECTION_MATERIALS:
if is_doi(d["value"]):
ctx.correction_dois.append((d["value"], correction))
else:
ctx.errors.append(
f"DOI validation failed. | details: doi={d['value']}"
)
continue
unique_dois.append(d)

for doi in reversed(unique_dois):
if not self.filter(doi):
Expand All @@ -132,9 +145,7 @@ def map_value(self, src_record, ctx, logger):
d for d in unique_dois if d.get("value", "").startswith(DATACITE_PREFIX)
]
other_dois = [
d
for d in unique_dois
if not d.get("value", "").startswith(DATACITE_PREFIX)
d for d in unique_dois if not d.get("value", "").startswith(DATACITE_PREFIX)
]

if len(cds_dois) > 1:
Expand Down Expand Up @@ -374,6 +385,20 @@ def map_value(self, src_record, ctx, logger):
}
)

for doi, correction in ctx.correction_dois:
identifiers.append(
{
"identifier": doi,
"scheme": "doi",
"relation_type": {"id": "issupplementedby"},
"resource_type": {
"id": INSPIRE_CORRECTION_MATERIALS[correction][
"resource_type"
]
},
}
)

identifiers.append(
{
"scheme": "inspire",
Expand Down
122 changes: 122 additions & 0 deletions site/tests/inspire_harvester/test_transformer.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@
"""INSPIRE harvester transformer tests."""
from unittest.mock import Mock, patch

import pytest
from edtf.parser.grammar import ParseException

from cds_rdm.inspire_harvester.logger import Logger
from cds_rdm.inspire_harvester.transform.context import MetadataSerializationContext
from cds_rdm.inspire_harvester.transform.mappers.article import ArticleFilesMapper
from cds_rdm.inspire_harvester.transform.mappers.basic_metadata import (
AdditionalDescriptionsMapper,
AdditionalTitlesMapper,
Expand Down Expand Up @@ -1540,3 +1542,123 @@ def test_transform_conference_mapper_fetch_fails(running_app):
}
]
assert ctx.errors == []


def _ctx():
return MetadataSerializationContext(
resource_type=ResourceType.ARTICLE, inspire_id="12345"
)


@pytest.mark.parametrize("material", ["erratum", "corrigendum", "addendum", "Erratum"])
@patch("cds_rdm.inspire_harvester.transform.mappers.identifiers.is_doi")
def test_transform_dois_correction_is_not_main(mock_is_doi, material, running_app):
"""Erratum/corrigendum/addendum DOIs never become the main DOI."""
mock_is_doi.return_value = True
src_record = {
"metadata": {
"dois": [
{"value": "10.1000/correction", "material": material},
{"value": "10.1000/main", "material": "publication"},
]
}
}
ctx = _ctx()

result = DOIMapper().map_value(src_record, ctx, Logger(inspire_id="12345"))

assert result["doi"]["identifier"] == "10.1000/main"
assert ctx.correction_dois == [("10.1000/correction", material.lower())]
assert ctx.extra_related_dois == []
assert not ctx.errors


@patch("cds_rdm.inspire_harvester.transform.mappers.identifiers.is_doi")
def test_transform_dois_correction_invalid(mock_is_doi, running_app):
"""An invalid correction DOI is reported and skipped."""
mock_is_doi.side_effect = lambda v: v != "bad"
src_record = {
"metadata": {
"dois": [
{"value": "10.1000/main"},
{"value": "bad", "material": "erratum"},
]
}
}
ctx = _ctx()

DOIMapper().map_value(src_record, ctx, Logger(inspire_id="12345"))

assert ctx.correction_dois == []
assert any("DOI validation failed" in e for e in ctx.errors)


@patch("cds_rdm.inspire_harvester.transform.mappers.identifiers.is_doi")
def test_transform_related_identifiers_correction_dois(mock_is_doi, running_app):
"""Correction DOIs become issupplementedby related identifiers."""
mock_is_doi.return_value = True
ctx = _ctx()
ctx.correction_dois.append(("10.1000/err", "erratum"))
ctx.correction_dois.append(("10.1000/add", "addendum"))
src_record = {"metadata": {"dois": [{"value": "10.1000/main"}]}}

result = RelatedIdentifiersMapper().map_value(
src_record, ctx, Logger(inspire_id="12345")
)

for doi in ("10.1000/err", "10.1000/add"):
assert {
"identifier": doi,
"scheme": "doi",
"relation_type": {"id": "issupplementedby"},
"resource_type": {"id": "publication-other"},
} in result


def _documents(*filenames, **extra):
return {
"metadata": {
"documents": [
{
"filename": name,
"key": f"key{i}",
"url": f"https://example.com/{i}",
**extra,
}
for i, name in enumerate(filenames)
]
}
}


def test_transform_files_tags_correction_files(running_app):
"""Correction files are tagged in their description; the filename is unchanged."""
src_record = _documents("Erratum.pdf", "document.pdf")

result = FilesMapper().map_value(src_record, _ctx(), Logger(inspire_id="12345"))

assert "default_preview" not in result
assert result["entries"]["Erratum.pdf"]["metadata"]["description"] == "erratum"
assert "description" not in result["entries"]["document.pdf"]["metadata"]


def test_transform_files_tags_correction_from_material(running_app):
"""The correction is detected from material as well as from the filename."""
src_record = _documents("a.pdf", "b.pdf")
src_record["metadata"]["documents"][0]["material"] = "Corrigendum"

result = FilesMapper().map_value(src_record, _ctx(), Logger(inspire_id="12345"))

assert result["entries"]["a.pdf"]["metadata"]["description"] == "corrigendum"
assert "description" not in result["entries"]["b.pdf"]["metadata"]


def test_article_files_mapper_keeps_correction_files(running_app):
"""Article files without source/material are dropped, corrections are kept."""
src_record = _documents("document.pdf", "erratum.pdf")

result = ArticleFilesMapper().map_value(
src_record, _ctx(), Logger(inspire_id="12345")
)

assert list(result["entries"]) == ["erratum.pdf"]
Loading
Loading