Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion cds_migrator_kit/rdm/collection_conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
"restricted",
"create_inclusion_request",
"update_new_version_publication_date",
"access_grants_view",
"preferred_model",
)

Expand Down
68 changes: 46 additions & 22 deletions cds_migrator_kit/rdm/records/load/entities/parent.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from flask import current_app
from invenio_access.permissions import system_identity
from invenio_accounts.models import User
from invenio_communities.communities.records.api import Community
from invenio_drafts_resources.services.records.uow import ParentRecordCommitOp
from invenio_pidstore.errors import PIDAlreadyExists
from invenio_pidstore.models import PersistentIdentifier, PIDStatus
Expand All @@ -31,20 +32,29 @@ class ParentLoad:
methods are the counterpart of ``RecordEntry``.
"""

def __init__(self, entry: MigrationEntry, migration_logger, record_state):
def __init__(
self,
entry: MigrationEntry,
migration_logger,
record_state,
is_split_public_record=None,
):
"""Constructor.

:param record_parent: the built ``RecordParent`` for this entry
(``MigrationEntry["parent"]``).
:param migration_logger: kept for parity with the other load-side
entity collaborators; not used directly by this class today.
:param is_split_public_record: whether this is the public component of a record split into a restricted/public pair. If None, this is not a split record.
"""
self.record_parent = entry["parent"]
self.migration_logger = migration_logger
self.record_state = record_state
self.migration_entry = entry
self.versions = entry["versions"]
self.ep_approval = entry.get("ep_approval")
self.is_split_public_record = is_split_public_record
self.has_restricted_community = False

def load(self, published_record, uow):
"""Load access/communities, then access grants for every version.
Expand All @@ -62,54 +72,68 @@ def load(self, published_record, uow):
freshly-read parent instance, which ``load_access_and_communities``'s
stale in-memory parent doesn't see).
"""
self.load_access_and_communities(published_record, uow)
self.load_access_grants(published_record, uow)
self.has_restricted_community = self.load_access_and_communities(
published_record, uow
)
self.load_access_grants(uow)
self.set_stateless_committee_approval(published_record, uow)

def load_access_and_communities(self, draft, uow):
"""Load access rights and communities in a single parent commit."""
parent = draft._record.parent
parent.access = self.record_parent.body["access"]
for community in self.record_parent.communities["ids"]:

has_restricted_community = False
for community_id in self.record_parent.communities["ids"]:
community = Community.get_record(community_id)
if community.access.visibility_is_restricted:
has_restricted_community = True
parent.communities.add(community)

parent.communities.default = self.record_parent.communities["default"]
uow.register(ParentRecordCommitOp(parent))
return has_restricted_community

def load_access_grants(self, published_record, uow):
def load_access_grants(self, uow):
"""Load access grants from metadata and record grants efficiently.

:param draft: the draft/published record whose parent grants are set.
:param recid: this record's legacy recid, for error reporting.
"""
recid = self.migration_entry["record"].recid
# `record_state["versions"]` is keyed by invenio's own sequential
# `record.versions.index` (always 1, 2, 3, ...), while `self.versions`
# (`entry["versions"]`) is keyed by the *legacy* version number, which
# can start above 1 when the legacy version 1 was hard-deleted (see
# ``RecordLoad.pre_publish``). Both lists are built by iterating the
# same records in the same order (``RecordLoad._load_versions``), so
# match them up by position rather than by (possibly mismatched) key.
legacy_versions = list(self.versions.keys())
for legacy_version, version_state in zip(
legacy_versions, self.record_state["versions"]
):
access_dict = self.versions[legacy_version]["access"]

for version_state in self.record_state["versions"]:
published_record = current_rdm_records_service.read(
system_identity, version_state["new_recid"]
)

if (
self.has_restricted_community
and published_record._record.access.protection.record == "public"
):
# We shouldn't get here (transform/entities/version.py should make the record fully restricted if it's in a restricted collection).
# However, we add an extra check anyway.
raise ManualImportRequired(
message="The record is public but it was added to at least one restricted community, which is not allowed.",
stage="load",
)

parent = published_record._record.parent
identity = system_identity

specific_file_restrictions = access_dict.get("meta", "")
if not specific_file_restrictions and not self.record_parent.access_grants:
file_restrictions = self.migration_entry["restrictions"]["files"]
if self.is_split_public_record or (
not file_restrictions and not self.record_parent.access_grants
):
# Skip loading access grants if this is the public component OR if the parent record doesn't have any
# custom grants and none of the files are restricted
continue

default_permission = "view"

groups, emails, grants_with_perms = self.record_parent.resolve_grants(
specific_file_restrictions
)
groups, emails, grants_with_perms = self.record_parent.resolve_grants()

# TODO: this is where the load steps actually starts, the above stuff is transforming
def _create_grant(subject_type, subject_id, permission):
grant_data = {
"grants": [
Expand Down
69 changes: 38 additions & 31 deletions cds_migrator_kit/rdm/records/load/entities/record.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
from cds_migrator_kit.rdm.records.transform.entities.parent import RecordParent
from cds_migrator_kit.rdm.records.transform.entities.record import RecordEntry
from cds_migrator_kit.rdm.records.transform.entities.version import (
VersionAccess,
VersionAccessObj,
VersionFileEntry,
)

Expand Down Expand Up @@ -85,7 +85,7 @@ def __init__(
record_entry: RecordEntry,
record_parent: RecordParent,
migration_logger,
is_final_record=True,
is_public_record=True,
update_new_version_publication_date=False,
record_state_logger=None,
):
Expand All @@ -96,7 +96,7 @@ def __init__(
:param record_parent: the built ``RecordParent`` for this entry -
only used on the first version, to set up the new draft's
parent access/communities (see ``pre_publish()``).
:param is_final_record: whether this is the "real"/final record for
:param is_public_record: whether this is the "real"/final record for
its legacy recid - False for the restricted half of an EP
approval split, which exists only as an internal source record
and must skip legacy-recid-facing side effects (rep-number PID
Expand All @@ -108,7 +108,7 @@ def __init__(
self.record_entry = record_entry
self.record_parent = record_parent
self.migration_logger = migration_logger
self.is_final_record = is_final_record
self.is_public_record = is_public_record
self.update_new_version_publication_date = update_new_version_publication_date
self.record_state_logger = record_state_logger

Expand Down Expand Up @@ -143,27 +143,34 @@ def load_files(
],
uow=uow,
)
path = file_data["eos_tmp_path"]
if file_is_empty(path):
raise ManualImportRequired(
message="Empty file detected; curator review required.",
field="files",
stage="load",
recid=recid,
priority="critical",
value=file_data["key"],
)
if pdf_has_zero_pages(path):
raise ManualImportRequired(
message=(
"Empty/corrupt PDF detected; curator review required."
),
field="files",
stage="load",
recid=recid,
priority="critical",
value=file_data["key"],
)

is_local_dev = current_app.config.get("CDS_MIGRATOR_KIT_ENV") == "local"

if not is_local_dev:
# Don't run the checks on the local dev env since files don't actually
# exist there, so the checks will fail.
path = file_data["eos_tmp_path"]
if file_is_empty(path):
raise ManualImportRequired(
message="Empty file detected; curator review required.",
field="files",
stage="load",
recid=recid,
priority="critical",
value=file_data["key"],
)
if pdf_has_zero_pages(path):
raise ManualImportRequired(
message=(
"Empty/corrupt PDF detected; curator review required."
),
field="files",
stage="load",
recid=recid,
priority="critical",
value=file_data["key"],
)

current_rdm_records_service.draft_files.set_file_content(
identity,
draft.id,
Expand Down Expand Up @@ -241,15 +248,15 @@ def _is_pdf(file_data):
record.files.default_preview = preview["key"]
uow.register(RecordCommitOp(record))

def load_access(self, draft, access_dict: VersionAccess, uow):
def load_access(self, draft, access_dict: VersionAccessObj, uow):
"""Set this version's access on the published record."""
record = draft._record
record.access = access_dict["access_obj"]
record.access = access_dict
uow.register(RecordCommitOp(record))

def assign_rep_numbers(self, draft):
"""Mint ``cdsrn`` PIDs for this draft's report-number identifiers."""
if not self.is_final_record:
if not self.is_public_record:
return
draft_report_nums = {}
for index, id in enumerate(draft.data["metadata"].get("identifiers", [])):
Expand Down Expand Up @@ -352,7 +359,7 @@ def pre_publish(self, identity, versions, version, draft, uow):

def after_publish_update_dois(self, identity, record, uow):
"""Update migrated DOIs post publish."""
if not self.is_final_record:
if not self.is_public_record:
return
# Skip versions that do not carry a DOI (e.g. older versions of a
# multi-version CDS DOI record - DOI lives only on the latest).
Expand Down Expand Up @@ -389,7 +396,7 @@ def after_publish_update_created(self, record, version_data, version, uow):

def after_publish_mint_recid(self, record):
"""Mint legacy ids for redirections assigned to the parent."""
if not self.is_final_record:
if not self.is_public_record:
return
legacy_recid = self.record_entry.recid
if record._record.versions.index == 1:
Expand Down Expand Up @@ -479,7 +486,7 @@ def log_record_state(self, record_state_context):
rolled back by a failure elsewhere in the same load (e.g. a grant
creation failure).
"""
if self.is_final_record:
if self.is_public_record:
self.record_state_logger.add_record_state(record_state_context)

def _load_record_state(self, legacy_recid, records):
Expand Down
Loading
Loading