diff --git a/.gitignore b/.gitignore index 415b3bf64..f11c4af3c 100644 --- a/.gitignore +++ b/.gitignore @@ -43,4 +43,6 @@ datavault-webapp/pids # ignore intellij run files .run/ TEMPLATES/* -dv5/local-db/docker/backup.D.SPEED.sql \ No newline at end of file +dv5/local-db/docker/backup.D.SPEED.sql +# this can set the java version for the Intellij IDE and will set java versions for terminals too if 'sdk config set sdkman_auto_env true' +.sdkmanrc \ No newline at end of file diff --git a/datavault-broker/src/main/java/org/datavaultplatform/broker/app/DataVaultBrokerApp.java b/datavault-broker/src/main/java/org/datavaultplatform/broker/app/DataVaultBrokerApp.java index a47c078a5..71683bf80 100644 --- a/datavault-broker/src/main/java/org/datavaultplatform/broker/app/DataVaultBrokerApp.java +++ b/datavault-broker/src/main/java/org/datavaultplatform/broker/app/DataVaultBrokerApp.java @@ -55,9 +55,9 @@ JacksonConfig.class, PropertiesConfig.class, EncryptionConfig.class, ActuatorConfig.class, ScheduleConfig.class, InitialiseConfig.class, SecurityActuatorConfig.class, SecurityConfig.class, ControllerConfig.class, - ServiceConfig.class, DatabaseConfig.class, + DatabaseConfig.class, LdapConfig.class, EmailConfig.class, EmailLocalConfig.class, RabbitConfig.class, - StorageClassNameResolverConfig.class, WebConfig.class + StorageClassNameResolverConfig.class, WebConfig.class, ServiceConfig.class }) @Slf4j //@EnableJSONDoc diff --git a/datavault-broker/src/main/java/org/datavaultplatform/broker/controllers/admin/AdminController.java b/datavault-broker/src/main/java/org/datavaultplatform/broker/controllers/admin/AdminController.java index 693f27844..5e9820cd9 100644 --- a/datavault-broker/src/main/java/org/datavaultplatform/broker/controllers/admin/AdminController.java +++ b/datavault-broker/src/main/java/org/datavaultplatform/broker/controllers/admin/AdminController.java @@ -10,6 +10,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.datavaultplatform.broker.queue.Sender; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.services.*; import org.datavaultplatform.common.PropNames; import org.datavaultplatform.common.event.Event; @@ -17,7 +18,6 @@ import org.datavaultplatform.common.model.*; import org.datavaultplatform.common.response.*; -import org.datavaultplatform.common.task.Task; import org.jsondoc.core.annotation.Api; import org.jsondoc.core.annotation.ApiHeader; import org.jsondoc.core.annotation.ApiHeaders; @@ -33,8 +33,6 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; -import com.fasterxml.jackson.databind.ObjectMapper; - import jakarta.servlet.http.HttpServletRequest; /** @@ -60,6 +58,7 @@ public class AdminController { private final ExternalMetadataService externalMetadataService; private final AuditsService auditsService; private final RolesAndPermissionsService permissionsService; + private final AdminDepositService adminDepositService; private final Sender sender; private final String optionsDir; private final String tempDir; @@ -73,7 +72,8 @@ public AdminController(VaultsService vaultsService, UsersService usersService, DepositsService depositsService, RetrievesService retrievesService, EventService eventService, ArchiveStoreService archiveStoreService, JobsService jobsService, ExternalMetadataService externalMetadataService, AuditsService auditsService, - RolesAndPermissionsService permissionsService, Sender sender, + RolesAndPermissionsService permissionsService, AdminDepositService adminDepositService, + Sender sender, @Value("${optionsDir:#{null}}") String optionsDir, @Value("${tempDir:#{null}}") String tempDir, @Value("${s3.bucketName:#{null}}") String bucketName, @@ -90,6 +90,7 @@ public AdminController(VaultsService vaultsService, UsersService usersService, this.externalMetadataService = externalMetadataService; this.auditsService = auditsService; this.permissionsService = permissionsService; + this.adminDepositService = adminDepositService; this.sender = sender; this.optionsDir = optionsDir; this.tempDir = tempDir; @@ -356,61 +357,10 @@ public ResponseEntity deleteDeposit(@RequestHeader(HEADER_USER_ID) Strin if (user == null) { throw new Exception("User '" + userID + "' does not exist"); } - - List jobs = deposit.getJobs(); - for (Job job : jobs) { - if (job.isError() == false && job.getState() != job.getStates().size() - 1) { - // There's an in-progress job for this deposit - throw new IllegalArgumentException("Job in-progress for this Deposit"); - } - } - - List archiveStores = archiveStoreService.getArchiveStores(); - if (archiveStores.isEmpty()) { - throw new Exception("No configured archive storage"); - } - LOGGER.info("Delete deposit archiveStores : {}", archiveStores); - archiveStores = this.addArchiveSpecificOptions(archiveStores); - - // Create a job to track this delete - Job job = new Job("org.datavaultplatform.worker.tasks.Delete"); - jobsService.addJob(deposit, job); - - // Ask the worker to process the data delete - try { - HashMap deleteProperties = new HashMap<>(); - deleteProperties.put(PropNames.DEPOSIT_ID, deposit.getID()); - deleteProperties.put(PropNames.BAG_ID, deposit.getBagId()); - deleteProperties.put(PropNames.ARCHIVE_SIZE, Long.toString(deposit.getArchiveSize())); - deleteProperties.put(PropNames.USER_ID, user.getID()); - deleteProperties.put(PropNames.NUM_OF_CHUNKS, Integer.toString(deposit.getNumOfChunks())); - for (Archive archive : deposit.getArchives()) { - deleteProperties.put(archive.getArchiveStore().getID(), archive.getArchiveId()); - } - - // Add a single entry for the user file storage - Map userFileStoreClasses = new HashMap<>(); - Map> userFileStoreProperties = new HashMap<>(); - //userFileStoreClasses.put(storageID, userStore.getStorageClass()); - //userFileStoreProperties.put(storageID, userStore.getProperties()); - - - Task deleteTask = new Task( - job, deleteProperties, archiveStores, - userFileStoreProperties, userFileStoreClasses, - null, null, - null, - null, null, - null, null, null); - ObjectMapper mapper = new ObjectMapper(); - String jsonDelete = mapper.writeValueAsString(deleteTask); - sender.send(jsonDelete); - } catch (Exception e) { - LOGGER.error("Exception while deleting a deposit", e); - } + adminDepositService.deleteDeposit(deposit, user); return new ResponseEntity<>(HttpStatus.OK); - } + private List addArchiveSpecificOptions(List archiveStores) { if (archiveStores != null && ! archiveStores.isEmpty()) { for (ArchiveStore archiveStore : archiveStores) { diff --git a/datavault-broker/src/main/java/org/datavaultplatform/broker/queue/EventListener.java b/datavault-broker/src/main/java/org/datavaultplatform/broker/queue/EventListener.java index 2f6b204f3..471d28eaa 100644 --- a/datavault-broker/src/main/java/org/datavaultplatform/broker/queue/EventListener.java +++ b/datavault-broker/src/main/java/org/datavaultplatform/broker/queue/EventListener.java @@ -35,6 +35,7 @@ import org.datavaultplatform.common.event.audit.ChunkAuditStarted; import org.datavaultplatform.common.event.delete.DeleteComplete; import org.datavaultplatform.common.event.delete.DeleteStart; +import org.datavaultplatform.common.event.delete.DeletedChunk; import org.datavaultplatform.common.event.deposit.ChunksDigestEvent; import org.datavaultplatform.common.event.deposit.Complete; import org.datavaultplatform.common.event.deposit.CompleteCopyUpload; @@ -59,6 +60,7 @@ import org.datavaultplatform.common.model.Retrieve; import org.datavaultplatform.common.model.User; import org.datavaultplatform.common.model.Vault; +import org.datavaultplatform.common.model.Archive; import org.springframework.amqp.core.Message; import org.springframework.amqp.core.MessageListener; import org.springframework.amqp.rabbit.annotation.RabbitListener; @@ -364,6 +366,8 @@ void processEvent(String messageBody, Event event, Deposit deposit, Job job) process28UploadedToUserStore(uploadedToUserStore); } else if (event instanceof UserStoreSpaceAvailableChecked userStoreSpaceAvailableChecked ){ process29UserStoreSpaceAvailableChecked(userStoreSpaceAvailableChecked); + } else if (event instanceof DeletedChunk deletedChunk ){ + process30DeletedChunk(deletedChunk); } else { throw new Exception( String.format("Failed to process unknown Event class[%s]message[%s]", event.getClass(), @@ -881,6 +885,18 @@ protected void process29UserStoreSpaceAvailableChecked(UserStoreSpaceAvailableCh ignore(event); } + protected void process30DeletedChunk(DeletedChunk deletedChunk) { + processDeposit(deletedChunk.getDeposit(), $deposit -> { + String archiveId = deletedChunk.getArchiveId(); + if (archiveId != null) { + Archive archive = archivesService.getArchiveByArchiveId(archiveId); + if (archive != null) { + deletedChunk.setArchive(archive); + } + } + }); + } + String getUserSubject(String type) { String userSubjectKey = USER_DEPOSIT_PREFIX + type; log.info("User Subject key: {}", userSubjectKey); diff --git a/datavault-broker/src/main/java/org/datavaultplatform/broker/scheduled/CheckForDelete.java b/datavault-broker/src/main/java/org/datavaultplatform/broker/scheduled/CheckForDelete.java index cadd6305a..510109e57 100644 --- a/datavault-broker/src/main/java/org/datavaultplatform/broker/scheduled/CheckForDelete.java +++ b/datavault-broker/src/main/java/org/datavaultplatform/broker/scheduled/CheckForDelete.java @@ -1,11 +1,8 @@ package org.datavaultplatform.broker.scheduled; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.datavaultplatform.broker.queue.Sender; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.services.*; -import org.datavaultplatform.common.PropNames; import org.datavaultplatform.common.model.*; -import org.datavaultplatform.common.task.Task; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; @@ -28,27 +25,17 @@ public class CheckForDelete implements ScheduledTask { private static final Logger log = LoggerFactory.getLogger(CheckForDelete.class); private final VaultsService vaultsService; - private final VaultsReviewService vaultsReviewService; + private final DepositsReviewService depositsReviewService; - private final ArchiveStoreService archiveStoreService; - private final RolesAndPermissionsService rolesAndPermissionsService; - private final UsersService usersService; - private final JobsService jobsService; - private final Sender sender; + private final AdminDepositService adminDepositService; @Autowired - public CheckForDelete(VaultsService vaultsService, VaultsReviewService vaultsReviewService, - DepositsReviewService depositsReviewService, ArchiveStoreService archiveStoreService, - RolesAndPermissionsService rolesAndPermissionsService, UsersService usersService, - JobsService jobsService, Sender sender) { + public CheckForDelete(VaultsService vaultsService, + DepositsReviewService depositsReviewService, + AdminDepositService adminDepositService) { this.vaultsService = vaultsService; - this.vaultsReviewService = vaultsReviewService; this.depositsReviewService = depositsReviewService; - this.archiveStoreService = archiveStoreService; - this.rolesAndPermissionsService = rolesAndPermissionsService; - this.usersService = usersService; - this.jobsService = jobsService; - this.sender = sender; + this.adminDepositService = adminDepositService; } @Override @@ -114,57 +101,6 @@ public void execute() throws Exception { // todo : move this method to a service class private void deleteDeposit(Deposit deposit) throws Exception { - log.info("Delete deposit with name " + deposit.getName()); - - List jobs = deposit.getJobs(); - for (Job job : jobs) { - if (job.isError() == false && job.getState() != job.getStates().size() - 1) { - // There's an in-progress job for this deposit - throw new IllegalArgumentException("Job in-progress for this Deposit"); - } - } - - List archiveStores = archiveStoreService.getArchiveStores(); - if (archiveStores.isEmpty()) { - throw new Exception("No configured archive storage"); - } - - log.info("Delete deposit archiveStores : {}", archiveStores); - archiveStores = archiveStoreService.addArchiveSpecificOptions(archiveStores); - - // Create a job to track this delete - Job job = new Job("org.datavaultplatform.worker.tasks.Delete"); - jobsService.addJob(deposit, job); - - // Ask the worker to process the data delete - - HashMap deleteProperties = new HashMap<>(); - deleteProperties.put(PropNames.DEPOSIT_ID, deposit.getID()); - deleteProperties.put(PropNames.BAG_ID, deposit.getBagId()); - deleteProperties.put(PropNames.ARCHIVE_SIZE, Long.toString(deposit.getArchiveSize())); - // We have no record of who requested the delete, is that acceptable? - deleteProperties.put(PropNames.USER_ID, null); - deleteProperties.put(PropNames.NUM_OF_CHUNKS, Integer.toString(deposit.getNumOfChunks())); - for (Archive archive : deposit.getArchives()) { - deleteProperties.put(archive.getArchiveStore().getID(), archive.getArchiveId()); - } - - // Add a single entry for the user file storage - Map userFileStoreClasses = new HashMap<>(); - Map> userFileStoreProperties = new HashMap<>(); - //userFileStoreClasses.put(storageID, userStore.getStorageClass()); - //userFileStoreProperties.put(storageID, userStore.getProperties()); - - Task deleteTask = new Task( - job, deleteProperties, archiveStores, - userFileStoreProperties, userFileStoreClasses, - null, null, - null, - null, null, - null, null, null); - ObjectMapper mapper = new ObjectMapper(); - String jsonDelete = mapper.writeValueAsString(deleteTask); - sender.send(jsonDelete); - + adminDepositService.deleteDeposit(deposit, null); } } diff --git a/datavault-broker/src/main/java/org/datavaultplatform/broker/services/AdminDepositService.java b/datavault-broker/src/main/java/org/datavaultplatform/broker/services/AdminDepositService.java new file mode 100644 index 000000000..a5fa3edf5 --- /dev/null +++ b/datavault-broker/src/main/java/org/datavaultplatform/broker/services/AdminDepositService.java @@ -0,0 +1,101 @@ +package org.datavaultplatform.broker.services; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.datavaultplatform.broker.queue.Sender; +import org.datavaultplatform.common.PropNames; +import org.datavaultplatform.common.model.*; +import org.datavaultplatform.common.task.Task; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Service +@Transactional +@ConditionalOnBean(Sender.class) +public class AdminDepositService { + + private static final Logger LOG = LoggerFactory.getLogger(AdminDepositService.class); + + private final ArchiveStoreService archiveStoreService; + private final JobsService jobsService; + private final Sender sender; + private final boolean workersSendDeletedChunkEvents; + + public AdminDepositService(ArchiveStoreService archiveStoreService, + JobsService jobsService, Sender sender, + @Value("${workers.send.deleted.chunk.events:false}") boolean workersSendDeletedChunkEvents) { + this.archiveStoreService = archiveStoreService; + this.jobsService = jobsService; + this.sender = sender; + this.workersSendDeletedChunkEvents = workersSendDeletedChunkEvents; + } + + public void deleteDeposit(Deposit deposit, User user) throws Exception { + final String userId = user == null ? null : user.getID(); + LOG.info("Delete deposit with name [{}] userId[{}]", deposit.getName(), userId); + + List jobs = deposit.getJobs(); + for (Job job : jobs) { + if (job.isError() == false && job.getState() != job.getStates().size() - 1) { + // There's an in-progress job for this deposit + throw new IllegalArgumentException("Job in-progress for this Deposit"); + } + } + + List archiveStores = archiveStoreService.getArchiveStores(); + if (archiveStores.isEmpty()) { + throw new Exception("No configured archive storage"); + } + + LOG.info("Delete deposit archiveStores : {}", archiveStores); + archiveStores = archiveStoreService.addArchiveSpecificOptions(archiveStores); + + // Create a job to track this delete + Job job = new Job("org.datavaultplatform.worker.tasks.Delete"); + jobsService.addJob(deposit, job); + + // Ask the worker to process the data delete + try { + + HashMap deleteProperties = new HashMap<>(); + deleteProperties.put(PropNames.DEPOSIT_ID, deposit.getID()); + deleteProperties.put(PropNames.BAG_ID, deposit.getBagId()); + deleteProperties.put(PropNames.ARCHIVE_SIZE, Long.toString(deposit.getArchiveSize())); + // NOTE : for scheduled deleted = the userId will be null + deleteProperties.put(PropNames.USER_ID, userId); + deleteProperties.put(PropNames.NUM_OF_CHUNKS, Integer.toString(deposit.getNumOfChunks())); + for (Archive archive : deposit.getArchives()) { + deleteProperties.put(archive.getArchiveStore().getID(), archive.getArchiveId()); + } + deleteProperties.put(PropNames.WORKERS_SEND_DELETED_CHUNK_EVENTS, + Boolean.toString(workersSendDeletedChunkEvents)); + + // Add a single entry for the user file storage + Map userFileStoreClasses = new HashMap<>(); + Map> userFileStoreProperties = new HashMap<>(); + //userFileStoreClasses.put(storageID, userStore.getStorageClass()); + //userFileStoreProperties.put(storageID, userStore.getProperties()); + + Task deleteTask = new Task( + job, deleteProperties, archiveStores, + userFileStoreProperties, userFileStoreClasses, + null, null, + null, + null, null, + null, null, null); + ObjectMapper mapper = new ObjectMapper(); + String jsonDelete = mapper.writeValueAsString(deleteTask); + sender.send(jsonDelete); + } catch (Exception e) { + LOG.error("Exception while deleting a deposit", e); + } + } +} diff --git a/datavault-broker/src/main/java/org/datavaultplatform/broker/services/ArchivesService.java b/datavault-broker/src/main/java/org/datavaultplatform/broker/services/ArchivesService.java index 05bbedc28..ce152e7e8 100644 --- a/datavault-broker/src/main/java/org/datavaultplatform/broker/services/ArchivesService.java +++ b/datavault-broker/src/main/java/org/datavaultplatform/broker/services/ArchivesService.java @@ -29,8 +29,12 @@ public List getArchives() { return archiveDAO.list(); } - public Archive getArchive(String archiveId) { - return archiveDAO.findById(archiveId).orElse(null); + public Archive getArchive(String id) { + return archiveDAO.findById(id).orElse(null); + } + + public Archive getArchiveByArchiveId(String archiveId) { + return archiveDAO.findByArchiveId(archiveId).orElse(null); } public void addArchive(Deposit deposit, ArchiveStore archiveStore, String archiveId) { diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java index 22a46c39b..8e381ff82 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/ActuatorTest.java @@ -4,6 +4,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.datavaultplatform.broker.app.DataVaultBrokerApp; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.queue.Sender; import org.datavaultplatform.broker.services.FileStoreService; import org.datavaultplatform.broker.test.AddTestProperties; @@ -59,6 +60,9 @@ public class ActuatorTest extends BaseDatabaseTest { @MockBean FileStoreService mFileStoreService; + @MockBean + AdminDepositService mAdminDepositService; + @Test void setup() { when(mFileStoreService.getFileStores()).thenReturn(Collections.emptyList()); diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/OpenApiBrokerTest.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/OpenApiBrokerTest.java index d98f5c73f..39394f119 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/OpenApiBrokerTest.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/actuator/OpenApiBrokerTest.java @@ -3,6 +3,7 @@ import io.swagger.v3.oas.models.OpenAPI; import lombok.extern.slf4j.Slf4j; import org.datavaultplatform.broker.app.DataVaultBrokerApp; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.queue.Sender; import org.datavaultplatform.broker.services.FileStoreService; import org.datavaultplatform.broker.test.AddTestProperties; @@ -46,6 +47,9 @@ public class OpenApiBrokerTest extends BaseDatabaseTest { @MockBean FileStoreService mFileStoreService; + + @MockBean + AdminDepositService mAdminDepositService; @Autowired OpenAPI openApi; diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/authentication/FileStoreControllerIT.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/authentication/FileStoreControllerIT.java index 51ae5dfc9..94f3c1211 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/authentication/FileStoreControllerIT.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/authentication/FileStoreControllerIT.java @@ -32,6 +32,7 @@ import org.datavaultplatform.broker.actuator.SftpFileStoreEndpoint; import org.datavaultplatform.broker.actuator.SftpFileStoreInfo; import org.datavaultplatform.broker.app.DataVaultBrokerApp; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.queue.Sender; import org.datavaultplatform.broker.test.AddTestProperties; import org.datavaultplatform.broker.test.BaseDatabaseTest; @@ -104,6 +105,8 @@ public class FileStoreControllerIT extends BaseDatabaseTest { String passphrase; @MockBean Sender sender; + @MockBean + AdminDepositService adminDepositService; @Autowired MockMvc mvc; @Autowired diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/config/InitialiseBeansConfigIT.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/config/InitialiseBeansConfigIT.java new file mode 100644 index 000000000..eca83c9bd --- /dev/null +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/config/InitialiseBeansConfigIT.java @@ -0,0 +1,117 @@ +package org.datavaultplatform.broker.config; + +import lombok.extern.slf4j.Slf4j; +import org.datavaultplatform.broker.app.DataVaultBrokerApp; +import org.datavaultplatform.broker.controllers.admin.AdminController; +import org.datavaultplatform.broker.queue.Sender; +import org.datavaultplatform.broker.services.AdminDepositService; +import org.datavaultplatform.broker.test.AddTestProperties; +import org.datavaultplatform.common.docker.DockerImage; +import org.datavaultplatform.common.util.UsesTestContainers; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.testcontainers.service.connection.ServiceConnection; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.MariaDBContainer; +import org.testcontainers.containers.RabbitMQContainer; +import org.testcontainers.containers.startupcheck.MinimumDurationRunningStartupCheckStrategy; +import org.testcontainers.junit.jupiter.Container; + +import java.time.Duration; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.datavaultplatform.broker.services.BaseEmailServiceTest.PORT_HTTP; +import static org.datavaultplatform.broker.services.BaseEmailServiceTest.PORT_SMTP; +import static org.datavaultplatform.common.ldap.BaseLDAPServiceIT.LDAP_ADMIN_PASSWORD; +import static org.datavaultplatform.common.ldap.BaseLDAPServiceIT.LDAP_EXPOSED_PORT; + +/* +Added this test to check on spring bean wiring. +When we use testcontainers for Rabbit, MariaDB, LDAP and EMAIL - will all the beans wire up. +There was a problem that AdminDepositService was not being created when the Broker was run as an application (not a test) +Note: AdminDepositService has '@ConditionalOnBean' - this is evaluated when the bean is first loaded, therefore, the order we +load java config files in DataVaultBrokerApp does matter. + */ +@SpringBootTest(classes = DataVaultBrokerApp.class) +@Slf4j +@AddTestProperties +@TestPropertySource(properties = { + "broker.controllers.enabled=true", + "broker.services.enabled=true", + "broker.scheduled.enabled=true", + "broker.rabbit.enabled=true", + "broker.database.enabled=true", + "auditdeposit.schedule=-", + "encryptioncheck.schedule=-", + "review.schedule=-", + "delete.schedule=-", + "retentioncheck.schedule=-"}) +@UsesTestContainers +class InitialiseBeansConfigIT { + + @Container + @ServiceConnection + // This container is once per class - not once per method. Methods can 'dirty' the database. + static final MariaDBContainer mariadb = new MariaDBContainer<>(DockerImage.MARIADB_IMAGE); + + @Container + @ServiceConnection + private static final RabbitMQContainer RABBIT = new RabbitMQContainer(DockerImage.RABBIT_IMAGE_NAME) + .withExposedPorts(5672,15672); + + @Container + private static final GenericContainer LDAP_CONTAINER = new GenericContainer<>(DockerImage.LDAP_IMAGE) + .withEnv("LDAP_ROOT", "o=myu.ed") + .withEnv("LDAP_ADMIN_PASSWORD", LDAP_ADMIN_PASSWORD) + //SCHEMA - allows 'eduniRefNo' attributes - via LDIF file + .withClasspathResourceMapping("ldap/eduniPersonSchema.ldif", "/schema/custom.ldif", + BindMode.READ_ONLY) + //USERS via LDIF file + .withClasspathResourceMapping("ldap/testUsers.ldif", "/custom/testUsers.ldif", + BindMode.READ_ONLY) + .withEnv("LDAP_CUSTOM_LDIF_DIR", "/custom") + .withExposedPorts(LDAP_EXPOSED_PORT) + .withStartupCheckStrategy( + //Gotta allow time for openldap to initialise + new MinimumDurationRunningStartupCheckStrategy(Duration.ofSeconds(5)) + ); + + + @Container + private static final GenericContainer MAILHOG_CONTAINER + = new GenericContainer<>(DockerImage.MAIL_IMAGE).withExposedPorts(PORT_SMTP, PORT_HTTP); + + @DynamicPropertySource + static void setupProperties(DynamicPropertyRegistry registry) { + setupMailProperties(registry); + } + + public static void setupMailProperties(DynamicPropertyRegistry registry) { + registry.add("tc.mailhog.http", () -> MAILHOG_CONTAINER.getMappedPort(PORT_HTTP)); + registry.add("mail.host", MAILHOG_CONTAINER::getHost); + registry.add("mail.port", () -> MAILHOG_CONTAINER.getMappedPort(PORT_SMTP)); + log.info("email http://localhost:{}", MAILHOG_CONTAINER.getMappedPort(PORT_HTTP)); + } + + @Autowired + AdminDepositService adminDepositService; + + @Autowired + AdminController adminController; + + @Autowired + Sender sender; + + @Test + void testBeans() { + assertThat(adminDepositService).isNotNull(); + assertThat(adminController).isNotNull(); + assertThat(sender).isNotNull(); + } + +} diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/config/MockServicesConfig.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/config/MockServicesConfig.java index 5c71f4780..d2fe5d143 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/config/MockServicesConfig.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/config/MockServicesConfig.java @@ -1,6 +1,7 @@ package org.datavaultplatform.broker.config; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.services.*; import org.springframework.boot.test.context.TestConfiguration; import org.springframework.boot.test.mock.mockito.MockBean; @@ -98,4 +99,6 @@ public class MockServicesConfig { @MockBean VaultsService mVaultsService; + @MockBean + AdminDepositService mAdminDepositService; } diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/DepositControllerIT.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/DepositControllerIT.java index 834319800..813c4ef67 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/DepositControllerIT.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/DepositControllerIT.java @@ -3,6 +3,7 @@ import lombok.extern.slf4j.Slf4j; import org.datavaultplatform.broker.app.DataVaultBrokerApp; import org.datavaultplatform.broker.config.MockRabbitConfig; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.services.*; import org.datavaultplatform.broker.test.AddTestProperties; import org.datavaultplatform.broker.test.BaseDatabaseTest; @@ -54,7 +55,10 @@ class DepositControllerIT extends BaseDatabaseTest { @MockBean EmailService emailService; - + + @MockBean + AdminDepositService mAdminDepositService; + @Autowired DepositsController controller; diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/GenerateDepositMessageTest.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/GenerateDepositMessageTest.java index 5123b5ecf..d4058fc06 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/GenerateDepositMessageTest.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/GenerateDepositMessageTest.java @@ -31,6 +31,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.junit.jupiter.MockitoExtension; +import org.skyscreamer.jsonassert.JSONAssert; @ExtendWith(MockitoExtension.class) @@ -45,7 +46,7 @@ public class GenerateDepositMessageTest extends BaseGenerateMessageTest { private static final String FILE_STORE_SRC_LABEL = "FILE_STORE-SRC-LABEL"; final File srcDir = new File(baseDir, "src"); - + @Captor ArgumentCaptor argMessage; private DepositsController dc; @@ -148,7 +149,9 @@ void testAddDepositToGenerateDepositMessage() { assertEquals(destPath, actualDestPath); JsonNode expected = mapper.readTree(getExpectedJson(bagId, srcPath, destPath)); - assertEquals(expected, convert(generated)); + JsonNode actual = convert(generated); + JSONAssert.assertEquals(expected.toPrettyString(), actual.toPrettyString(), true); + assertEquals(expected, actual); log.info("Generated Message {}", expected.toPrettyString()); log.info("END SENT MESSAGE"); } @@ -192,11 +195,11 @@ private String getExpectedJson(String bagId, String srcRoot, String destRoot) { + " \"userFileStoreClasses\" : {" + " \"FILE-STORE-SRC-ID\" : \"org.datavaultplatform.common.storage.impl.LocalFileSystem\"" + " }," - + " \"chunkFilesDigest\" : null," + + " \"chunkFilesDigest\" : {}," + " \"tarIV\" : null," - + " \"chunksIVs\" : null," + + " \"chunksIVs\" : {}," + " \"encTarDigest\" : null," - + " \"encChunksDigest\" : null," + + " \"encChunksDigest\" : {}," + " \"lastEvent\" : null," + " \"chunksToAudit\" : null," + " \"archiveIds\" : null," diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/RetrieveRestartIT.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/RetrieveRestartIT.java index 388f2bafc..79be2235c 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/RetrieveRestartIT.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/controllers/RetrieveRestartIT.java @@ -6,6 +6,7 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.datavaultplatform.broker.app.DataVaultBrokerApp; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.email.EmailBodyGenerator; import org.datavaultplatform.broker.queue.MessageIdProcessedListener; import org.datavaultplatform.broker.services.*; @@ -152,6 +153,9 @@ void init() { @MockBean MessageIdProcessedListener mMessageIdProcessedListener; + @MockBean + AdminDepositService mAdminDepositService; + List processedMessageIds; protected org.datavaultplatform.common.model.ArchiveStore archiveStore; diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/queue/EventListenerIT.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/queue/EventListenerIT.java index e07222dab..078195280 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/queue/EventListenerIT.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/queue/EventListenerIT.java @@ -6,9 +6,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import java.time.LocalDate; -import java.util.Base64; -import java.util.Date; -import java.util.List; +import java.util.*; +import java.util.stream.Stream; + import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.datavaultplatform.broker.app.DataVaultBrokerApp; @@ -25,6 +25,7 @@ import org.datavaultplatform.common.event.audit.ChunkAuditStarted; import org.datavaultplatform.common.event.delete.DeleteComplete; import org.datavaultplatform.common.event.delete.DeleteStart; +import org.datavaultplatform.common.event.delete.DeletedChunk; import org.datavaultplatform.common.event.deposit.Complete; import org.datavaultplatform.common.event.deposit.CompleteCopyUpload; import org.datavaultplatform.common.event.deposit.ComputedChunks; @@ -45,7 +46,6 @@ import org.junit.jupiter.params.provider.ValueSource; import org.springframework.amqp.rabbit.listener.RabbitListenerEndpointRegistry; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.Import; @@ -65,8 +65,10 @@ @Import({EventListener.class, TaskTimerSupport.class}) @Slf4j @TestMethodOrder(MethodOrderer.MethodName.class) -public class EventListenerIT extends BaseDatabaseTest { +class EventListenerIT extends BaseDatabaseTest { + private static final String TEST_ARCHIVE_ID = "TEST-ARCHIVE_ID"; + @MockBean EmailService emailService; @@ -100,6 +102,15 @@ public class EventListenerIT extends BaseDatabaseTest { @Autowired AuditsService auditsService; + @Autowired + EventService eventService; + + @Autowired + ArchivesService archivesService; + + @Autowired + ArchiveStoreService archiveStoreService; + @MockBean RabbitListenerEndpointRegistry registry; private final String userId = "user123"; @@ -124,8 +135,6 @@ public class EventListenerIT extends BaseDatabaseTest { Audit audit; Group group; - @Autowired - private EventService eventService; @BeforeEach void setup(){ @@ -208,6 +217,16 @@ void setup(){ assertThat(retrieve.getID()).isNotNull(); } + Optional getLastJobEvent(String jobID) { + List allEvents = eventService.getEvents(); + Stream jobEvents = allEvents.stream() + .filter(ev -> jobID.equals(ev.getJob().getID())); + Optional lastJobEvent = jobEvents + .sorted(Comparator.comparing(Event::getSequence)) //sort by sequence number ascending + .reduce((first, second) -> second); //this is a trick to get the last event + return lastJobEvent; + } + @Test void test00EventListener() { assertNotNull(eventListener); @@ -274,6 +293,7 @@ void test02v1updateProgress() { + " \"agentType\": \"WORKER\"" + " }"; Event event = eventListener.onMessageInternal(message); + assertThat(event).isInstanceOf(UpdateProgress.class); } @SneakyThrows @@ -521,6 +541,8 @@ void test10Error() { + " \"timestamp\": \"2022-09-16T15:12:40.152Z\"," + " \"sequence\": 36," + " \"persistent\": true," + + " \"chunkNumber\": 123," + + " \"message\": \"the error message\"," + " \"depositId\": \"" + depositId + "\"," + " \"vaultId\" : \"" + vaultId + "\"," + " \"jobId\" : \"" + jobGenericId + "\"," @@ -530,6 +552,20 @@ void test10Error() { + " }"; Event event = eventListener.onMessageInternal(message); assertEquals(org.datavaultplatform.common.event.Error.class, event.getClass()); + + // double check that we have saved the Error event to the database by fetching it and checking it + + org.datavaultplatform.common.event.Error error = (org.datavaultplatform.common.event.Error) event; + assertEquals(123, error.getChunkNumber()); + assertEquals("the error message", error.getMessage()); + + Optional lastDepositJobEventOpt = getLastJobEvent(jobGenericId); + Event lastDepositEvent = lastDepositJobEventOpt.orElseThrow(); + assertThat(lastDepositEvent).isEqualTo(event); + assertThat(lastDepositEvent.getJob()).isEqualTo(event.getJob()); + assertThat(lastDepositEvent.getDeposit()).isEqualTo(event.getDeposit()); + assertThat(lastDepositEvent.getMessage()).isEqualTo(event.getMessage()); + assertThat(lastDepositEvent.getChunkNumber()).isEqualTo(event.getChunkNumber()); } @Nested @@ -736,8 +772,8 @@ void test29UserStoreSpaceAvailableChecked() { }) @SneakyThrows void testRetrieveError(String eventClass) { - Class clazz = Class.forName(eventClass); - assertThat(Event.class.isAssignableFrom(clazz)); + Class clazz = Class.forName(eventClass); + assertThat(Event.class).isAssignableFrom(clazz); String message = "{" + " \"message\": \"CUSTOM ERROR MESSAGE\"," + " \"eventClass\": \"" + eventClass + "\"," @@ -1065,4 +1101,89 @@ void test24ValidationComplete() { Event event = eventListener.onMessageInternal(message); assertEquals(ValidationComplete.class, event.getClass()); } + + @Test + @SneakyThrows + void test30DeletedChunk() { + ArchiveStore archiveStore = new ArchiveStore(); + archiveStoreService.addArchiveStore(archiveStore); + + archivesService.addArchive(this.deposit, archiveStore, TEST_ARCHIVE_ID); + + assertThat(archivesService.getArchiveByArchiveId(TEST_ARCHIVE_ID)).isNotNull(); + String message = "{" + + " \"message\" : \"Deleted Chunk [7/10] from (MultiLocationsArchiveStoreSuccessImpl/TEST-ARCHIVE-STORE-ID//private/tmp/delete/location-one)\"," + + " \"eventClass\" : \"org.datavaultplatform.common.event.delete.DeletedChunk\"," + + " \"timestamp\" : \"2026-02-03T15:05:08.385Z\"," + + " \"sequence\" : 123," + + " \"persistent\" : true," + + " \"depositId\" : \"" + depositId + "\"," + + " \"jobId\" : \"" + jobDepositId + "\"," + + " \"userId\" : \"" + userId + "\"," + + " \"agent\" : \"datavault-worker-1\"," + + " \"agentType\" : \"WORKER\"," + + " \"archiveId\" : \"" + TEST_ARCHIVE_ID + "\"," + + " \"location\" : \"/private/tmp/delete/location-one\"," + + " \"assigneeId\" : null," + + " \"chunkNumber\" : 123," + + " \"archiveStoreId\" : \"TEST-ARCHIVE-STORE-ID\"" + + "}"; + + Event event = eventListener.onMessageInternal(message); + assertEquals(DeletedChunk.class, event.getClass()); + DeletedChunk dc = (DeletedChunk) event; + assertThat(dc.getID()) + .withFailMessage("ID is null") + .isNotNull(); + // DEPOSIT + assertThat(dc.getDeposit()) + .withFailMessage("Deposit is null") + .isNotNull(); + assertThat(dc.getDepositId()) + .withFailMessage("DepositId is null") + .isNotNull(); + assertThat(dc.getJob()) + .withFailMessage("Job is null") + .isNotNull(); + assertThat(dc.getJobId()) + .withFailMessage("JobId is null") + .isNotNull(); + // USER + assertThat(dc.getUser()) + .withFailMessage("User is null") + .isNotNull(); + assertThat(dc.getUserId()) + .withFailMessage("UserId is null") + .isNotNull(); + // AGENT + assertThat(dc.getAgent()) + .withFailMessage("Agent is null") + .isNotNull(); + // Archive + assertThat(dc.getArchive()) + .withFailMessage("Archive is null") + .isNotNull(); + assertThat(dc.getArchiveId()) + .withFailMessage("ArchiveId is null") + .isNotNull(); + // ArchiveStoreId + assertThat(dc.getArchiveStoreId()) + .withFailMessage("ArchiveStoreId is null") + .isEqualTo("TEST-ARCHIVE-STORE-ID"); + // AgentType + assertThat(dc.getAgentType()) + .withFailMessage("AgentType is null") + .isNotNull(); + // VAULT + assertThat(dc.getVault()) + .withFailMessage("Vault is NOT NULL") + .isNull(); + assertThat(dc.getVaultId()) + .withFailMessage("VaultId is NOT NULL") + .isNull(); + assertThat(dc.getChunkNumber()) + .isEqualTo(123); + assertThat(dc.getLocation()) + .isEqualTo("/private/tmp/delete/location-one"); + } } diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/queue/EventListenerTest.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/queue/EventListenerTest.java index 1d66f93c2..b6d25cb03 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/queue/EventListenerTest.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/queue/EventListenerTest.java @@ -2170,7 +2170,7 @@ public String getDigestAlgorithm() { } }; assertEquals(0, deposit.getNumOfChunks()); - assertNull(deposit.getDepositChunks()); + assertThat(deposit.getDepositChunks().isEmpty()); doNothing().when(depositsService).updateDeposit(deposit); sut.updateDepositWithChunks(deposit, event); diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/scheduled/ScheduledTasksWithDbIT.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/scheduled/ScheduledTasksWithDbIT.java index 269789ea4..1ce3ea92c 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/scheduled/ScheduledTasksWithDbIT.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/scheduled/ScheduledTasksWithDbIT.java @@ -8,6 +8,7 @@ import lombok.extern.slf4j.Slf4j; import org.datavaultplatform.broker.app.DataVaultBrokerApp; import org.datavaultplatform.broker.config.MockRabbitConfig; +import org.datavaultplatform.broker.services.AdminDepositService; import org.datavaultplatform.broker.services.EmailService; import org.datavaultplatform.broker.test.AddTestProperties; import org.datavaultplatform.broker.test.BaseReuseDatabaseTest; @@ -68,6 +69,9 @@ public class ScheduledTasksWithDbIT extends BaseReuseDatabaseTest { @MockBean EmailService emailService; + @MockBean + AdminDepositService adminDepositService; + @Autowired AuditDepositsChunks scheduled1auditDepositsChunks; diff --git a/datavault-broker/src/test/java/org/datavaultplatform/broker/services/VaultsServiceIT.java b/datavault-broker/src/test/java/org/datavaultplatform/broker/services/VaultsServiceIT.java index 387737f5f..04ff4496c 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/broker/services/VaultsServiceIT.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/broker/services/VaultsServiceIT.java @@ -14,6 +14,7 @@ import org.junit.jupiter.api.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; import org.springframework.context.annotation.Import; import org.springframework.test.context.TestPropertySource; @@ -32,6 +33,9 @@ public class VaultsServiceIT extends BaseReuseDatabaseTest { @Autowired private RolesAndPermissionsService rolesAndPermissionsService; + @MockBean + AdminDepositService adminDepositService; + @Test public void checkVaultCount() { RoleAssignment isAdminRoleAssignment = new RoleAssignment(); diff --git a/datavault-broker/src/test/java/org/datavaultplatform/common/model/dao/EventDAOIT.java b/datavault-broker/src/test/java/org/datavaultplatform/common/model/dao/EventDAOIT.java index 6a0465bcc..72f9bba95 100644 --- a/datavault-broker/src/test/java/org/datavaultplatform/common/model/dao/EventDAOIT.java +++ b/datavault-broker/src/test/java/org/datavaultplatform/common/model/dao/EventDAOIT.java @@ -15,16 +15,20 @@ import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.datavaultplatform.broker.app.DataVaultBrokerApp; +import org.datavaultplatform.broker.services.EventService; import org.datavaultplatform.broker.test.AddTestProperties; import org.datavaultplatform.broker.test.BaseDatabaseTest; import org.datavaultplatform.broker.test.TestUtils; import org.datavaultplatform.common.event.Event; +import org.datavaultplatform.common.event.delete.DeletedChunk; import org.datavaultplatform.common.event.deposit.*; import org.datavaultplatform.common.event.Error; import org.datavaultplatform.common.event.retrieve.*; +import org.datavaultplatform.common.model.Agent; import org.datavaultplatform.common.model.Deposit; import org.datavaultplatform.common.model.Job; import org.datavaultplatform.common.model.Vault; +import org.datavaultplatform.common.storage.impl.LocalFileSystem; import org.datavaultplatform.common.util.RetrievedChunks; import org.datavaultplatform.common.util.StoredChunks; import org.junit.jupiter.api.AfterEach; @@ -65,7 +69,10 @@ public class EventDAOIT extends BaseDatabaseTest { @PersistenceContext EntityManager em; - + + @Autowired + private EventService eventService; + @Nested class BlobTests { @@ -778,4 +785,63 @@ RetrieveComplete get06RetrieveComplete(String jobId, String depositId, String re return new RetrieveComplete(jobId, depositId, retrieveId); } } + + /** + * Tests that the Deposit and Job associated with a DeletedChunk event are correctly handled + * by Hibernate's session cache/identity map. + * Verifies that when retrieving a DeletedChunk event, the associated Deposit and Job entities + * are the same instances as those expected. + * We have to use EntityManager::flush to force saving to DB as generally DB writes happen at end of transaction. + */ + @Transactional + @Test + void testDepositAndJobLazyLoadingOfDeletedChunkEvent() { + + Deposit deposit = new Deposit(); + deposit.setHasPersonalData(false); + deposit.setName("test-deposit-name"); + depositDAO.save(deposit); + + Job job = new Job(); + job.setDeposit(deposit); + jobDAO.save(job); + + em.flush(); + em.clear(); + assertThat(em.contains(deposit)).isFalse(); + assertThat(em.contains(job)).isFalse(); + + DeletedChunk dc = new DeletedChunk(job.getID(), deposit.getID(), 123, 999, LocalFileSystem.class, "TST-ARCHIVE-STORE-ID", "TST-LOCATION"); + dc.setAgent("TST-AGENT"); + dc.setAgentType(Agent.AgentType.WORKER); + dc.setMessage("test-message"); + dc.setJob(job); + dc.setDeposit(deposit); + + eventService.addEvent(dc); + deposit.getEvents().add(dc); + + em.flush(); + em.clear(); + + // get back the job and deposit from db - check that have correct ids + Deposit depoFromDb = depositDAO.getReferenceById(deposit.getID()); + Job jobFromDeposit = depoFromDb.getJobs().get(0); + assertThat(depoFromDb.getID()).isEqualTo(deposit.getID()); + assertThat(jobFromDeposit.getID()).isEqualTo(job.getID()); + assertThat(em.contains(depoFromDb)).isTrue(); + assertThat(em.contains(jobFromDeposit)).isTrue(); + + // get back event from db - check it has correct id and Deposit and Job are existing hibernate objects + Event eventFromDb = eventService.getEvent(dc.getID()); + assertThat(eventFromDb.getID()).isEqualTo(dc.getID()); + + assertThat(em.contains(eventFromDb)).isTrue(); + + assertThat(eventFromDb.getDeposit()).isSameAs(depoFromDb); + assertThat(eventFromDb.getJob()).isSameAs(jobFromDeposit); + + Event depositEvent1 = depoFromDb.getEvents().get(0); + assertThat(depositEvent1).isSameAs(eventFromDb); + } } diff --git a/datavault-broker/src/test/resources/samples/sampleDeleteChunkError.json b/datavault-broker/src/test/resources/samples/sampleDeleteChunkError.json new file mode 100644 index 000000000..10650ad9d --- /dev/null +++ b/datavault-broker/src/test/resources/samples/sampleDeleteChunkError.json @@ -0,0 +1,27 @@ +{ + "id": null, + "message": "Deposit delete failed: ArchiveStore[ArchiveStoreFailureImpl/TEST-ARCHIVE-STORE-ID]Location[no-location]ChunkNum[1]Cause[java.lang.RuntimeException/oops@1]", + "retrieveId": null, + "eventClass": "org.datavaultplatform.common.event.Error", + "nextState": null, + "timestamp": "2026-01-30T10:19:30.443Z", + "sequence": 0, + "persistent": true, + "depositId": "TEST-DEPOSIT-ID", + "vaultId": null, + "jobId": "TEST-JOB-ID", + "userId": "TEST-USER-ID", + "agent": null, + "remoteAddress": null, + "userAgent": null, + "agentType": null, + "auditId": null, + "archiveId": "TEST-ARCHIVE-ID", + "location": "no-location", + "chunkId": null, + "assigneeId": null, + "schoolId": null, + "roleId": null, + "chunkNumber": 1, + "archiveStoreId": "TEST-ARCHIVE-STORE-ID" +} \ No newline at end of file diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/PropNames.java b/datavault-common/src/main/java/org/datavaultplatform/common/PropNames.java index fda19d984..9390b7699 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/PropNames.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/PropNames.java @@ -57,4 +57,5 @@ public interface PropNames { String USER_FS_RETRY_DELAY_MS_1 = "userFsRetryDelayMs1"; String USER_FS_RETRY_DELAY_MS_2 = "userFsRetryDelayMs2"; String NON_RESTART_JOB_ID = "nonRestartJobId"; + String WORKERS_SEND_DELETED_CHUNK_EVENTS = "workersSendDeletedChunkEvents"; } diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/event/Event.java b/datavault-common/src/main/java/org/datavaultplatform/common/event/Event.java index ef0388bf3..71f01d3c3 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/event/Event.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/event/Event.java @@ -33,6 +33,7 @@ @JsonSubTypes.Type(value = DeleteStart.class, name = "org.datavaultplatform.common.event.delete.DeleteStart"), @JsonSubTypes.Type(value = DeleteComplete.class, name = "org.datavaultplatform.common.event.delete.DeleteComplete"), + @JsonSubTypes.Type(value = DeletedChunk.class, name = "org.datavaultplatform.common.event.delete.DeletedChunk"), @JsonSubTypes.Type(value = ValidationComplete.class, name = "org.datavaultplatform.common.event.deposit.ValidationComplete"), @JsonSubTypes.Type(value = ComputedSize.class, name = "org.datavaultplatform.common.event.deposit.ComputedSize"), @@ -81,7 +82,7 @@ @JsonSubTypes.Type(value = Event.class, name = "org.datavaultplatform.common.event.Event"), @JsonSubTypes.Type(value = Error.class, name = "org.datavaultplatform.common.event.Error"), @JsonSubTypes.Type(value = InitStates.class, name = "org.datavaultplatform.common.event.InitStates"), - @JsonSubTypes.Type(value = UpdateProgress.class, name = "org.datavaultplatform.common.event.UpdateProgress") + @JsonSubTypes.Type(value = UpdateProgress.class, name = "org.datavaultplatform.common.event.UpdateProgress"), }) @JsonIgnoreProperties(ignoreUnknown = true) @Entity diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/event/delete/DeletedChunk.java b/datavault-common/src/main/java/org/datavaultplatform/common/event/delete/DeletedChunk.java new file mode 100644 index 000000000..b80d23850 --- /dev/null +++ b/datavault-common/src/main/java/org/datavaultplatform/common/event/delete/DeletedChunk.java @@ -0,0 +1,33 @@ +package org.datavaultplatform.common.event.delete; + +import jakarta.persistence.Entity; + +import org.datavaultplatform.common.event.Event; +import org.datavaultplatform.common.storage.ArchiveStore; + +@Entity +public class DeletedChunk extends Event { + + public DeletedChunk() { + } + + public DeletedChunk(String jobId, String depositId, + int chunkNumber, int numberOfChunks, + Class archiveType, String archiveStoreId, String location) { + super("Deleted Chunk [%d/%d] from (%s/%s/%s)" + .formatted(chunkNumber, numberOfChunks, archiveType.getSimpleName(), archiveStoreId, location)); + this.setEventClass(DeletedChunk.class.getCanonicalName()); + this.setDepositId(depositId); + this.setJobId(jobId); + this.setChunkNumber(chunkNumber); + this.setArchiveStoreId(archiveStoreId); + + // location is not persisted to database - that's why it's also in the message + this.setLocation(location); + } + + @Override + public String toString() { + return getMessage(); + } +} diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/model/Deposit.java b/datavault-common/src/main/java/org/datavaultplatform/common/model/Deposit.java index 1ca0612f4..23f25bd73 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/model/Deposit.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/model/Deposit.java @@ -64,33 +64,33 @@ public class Deposit implements Identified { @JsonIgnore @OneToMany(targetEntity=Event.class, mappedBy="deposit", fetch=FetchType.LAZY) @OrderBy("timestamp, sequence") - private List events; + private List events = new ArrayList<>(); // A Deposit can have a number of deposit paths @OneToMany(targetEntity=DepositPath.class, mappedBy="deposit", fetch=FetchType.LAZY, cascade = CascadeType.ALL) - private List depositPaths; + private List depositPaths = new ArrayList<>(); // A Deposit can have a number of deposit chunks @OneToMany(targetEntity=DepositChunk.class, mappedBy="deposit", fetch=FetchType.LAZY, cascade = CascadeType.ALL) - private List depositChunks; + private List depositChunks = new ArrayList<>(); // A Deposit can have a number of active jobs @JsonIgnore @OneToMany(targetEntity=Job.class, mappedBy="deposit", fetch=FetchType.LAZY) @OrderBy("timestamp") - private List jobs; + private List jobs = new ArrayList<>(); // A Deposit can have a number of retrieves @JsonIgnore @OneToMany(targetEntity=Retrieve.class, mappedBy="deposit", fetch=FetchType.LAZY) @OrderBy("timestamp") - private List retrieves; + private List retrieves = new ArrayList<>(); // A Deposit can have a number of reviews @JsonIgnore @OneToMany(targetEntity=DepositReview.class, mappedBy="deposit", fetch=FetchType.LAZY) @OrderBy("creationTime") - private List depositReviews; + private List depositReviews = new ArrayList<>(); @ApiObjectField(description = "Status of the Deposit", allowedvalues={"NOT_STARTED", "IN_PROGRESS", "COMPLETE"}) diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/model/dao/ArchiveDAO.java b/datavault-common/src/main/java/org/datavaultplatform/common/model/dao/ArchiveDAO.java index be0061189..235c7aaaf 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/model/dao/ArchiveDAO.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/model/dao/ArchiveDAO.java @@ -30,4 +30,7 @@ public interface ArchiveDAO extends BaseDAO { LIMIT 1 """) Optional findLatestByDepositIdAndArchiveStoreId(String depositId, String archiveStoreId); + + @EntityGraph(Archive.EG_ARCHIVE) + Optional findByArchiveId(String archiveId); } diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/FileSystemUtils.java b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/FileSystemUtils.java index 871f3fa98..47948f87b 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/FileSystemUtils.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/storage/impl/FileSystemUtils.java @@ -17,6 +17,9 @@ public class FileSystemUtils { private static final String EMPTY_STRING = ""; + private FileSystemUtils() { + } + public static Path getAbsolutePath(String filePath, String location) { // Join the requested path to the root of the filesystem. @@ -73,7 +76,16 @@ public static long getUsableSpace(File file) { public static void delete(String path, String location) throws Exception { Path absolutePath = getAbsolutePath(path, location); - Files.deleteIfExists(absolutePath); + log.info("Starting to Delete [{}] ", absolutePath); + boolean deleted = false; + try { + deleted = Files.deleteIfExists(absolutePath); + } catch(Exception ex) { + log.info("Attempt to Delete [{}] failed ", absolutePath, ex); + throw ex; + } finally { + log.info("Attempt to Delete [{}] success ? {}", absolutePath, deleted); + } } public static String store(String path, File working, Progress progress, String location) throws Exception{ diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/task/TaskExecutor.java b/datavault-common/src/main/java/org/datavaultplatform/common/task/TaskExecutor.java index 0876ccdc7..c8aa471f8 100644 --- a/datavault-common/src/main/java/org/datavaultplatform/common/task/TaskExecutor.java +++ b/datavault-common/src/main/java/org/datavaultplatform/common/task/TaskExecutor.java @@ -1,6 +1,8 @@ package org.datavaultplatform.common.task; import org.datavaultplatform.common.util.Utils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.util.Assert; import java.util.ArrayList; @@ -8,12 +10,13 @@ import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; -import java.util.stream.Collectors; public class TaskExecutor { - private final int numThreads; - private final String errorLabel; + private static final Logger LOG = LoggerFactory.getLogger(TaskExecutor.class); + private static final long TIMEOUT_MINUTES = 5; + private final int numThreads; + private final String errorLabel; private final List> tasks = new ArrayList<>(); private final List> origTasks = new ArrayList<>(); @@ -52,31 +55,66 @@ private Callable wrap(Callable task) { public synchronized void execute() throws Exception { execute(result -> {}); } - - public synchronized void execute(Consumer consumer) throws Exception { - if (executed.getAndSet(true)) { - throw new IllegalStateException("Already executed"); - } - ExecutorService service = Executors.newFixedThreadPool(numThreads); + public synchronized void execute(Consumer consumer) throws Exception { + if (executed.getAndSet(true)) { + throw new IllegalStateException("Already executed"); + } - List> futures = tasks.stream() - .map(service::submit) - .toList(); + ExecutorService service = Executors.newFixedThreadPool(numThreads); - service.shutdown(); + List> futures = tasks.stream() + .map(service::submit) + .toList(); + try { + // shutdown - prevents the submission of further Callable Tasks + service.shutdown(); - // service.awaitTermination(1, TimeUnit.MINUTES); + for (Future future : futures) { + getResultFromFuture(future, consumer); + } - for (Future future : futures) { - try { - T result = future.get(); - consumer.accept(result); - } catch (ExecutionException ee) { - Utils.handleExecutionException(ee, errorLabel); - } + } catch (InterruptedException e) { + LOG.error("Main thread interrupted!"); + // Restore the status so the calling code knows we were interrupted + Thread.currentThread().interrupt(); + } finally { + handleShutdown(service); + } } - service.shutdownNow(); - } + private void getResultFromFuture(Future future, Consumer consumer) throws Exception { + try { + T result = future.get(); + consumer.accept(result); + } catch (ExecutionException ee) { + Utils.handleExecutionException(ee, errorLabel); + } + } + + private void handleShutdown(ExecutorService executor) { + // If it's already fully closed, we're done. + if (executor == null || executor.isTerminated()) return; + + try { + // Only call shutdown if it hasn't been called yet + if (!executor.isShutdown()) { + executor.shutdown(); + } + + boolean success = executor.awaitTermination(TIMEOUT_MINUTES, TimeUnit.MINUTES); + LOG.info("Tasks Finished within [{}] minute timeout ? {}", TIMEOUT_MINUTES, success); + + } catch (InterruptedException ie) { + LOG.warn("Shutdown interrupted, forcing immediate termination."); + Thread.currentThread().interrupt(); + } finally { + // If the 5 minute passed OR we were interrupted, + // and tasks are STILL running, we kill them now. + if (!executor.isTerminated()) { + List notStarted = executor.shutdownNow(); + LOG.warn("ExecutorService[{}]Terminated. NotStartedCount[{}]", errorLabel, notStarted.size()); + } + } + } } diff --git a/datavault-common/src/test/java/org/datavaultplatform/common/ldap/BaseLDAPServiceIT.java b/datavault-common/src/test/java/org/datavaultplatform/common/ldap/BaseLDAPServiceIT.java index 045249bf2..dc5d2c962 100644 --- a/datavault-common/src/test/java/org/datavaultplatform/common/ldap/BaseLDAPServiceIT.java +++ b/datavault-common/src/test/java/org/datavaultplatform/common/ldap/BaseLDAPServiceIT.java @@ -42,7 +42,7 @@ public abstract class BaseLDAPServiceIT { */ public static final String LDAP_ADMIN_PASSWORD = "test-password"; - private static final int LDAP_EXPOSED_PORT = 1389; + public static final int LDAP_EXPOSED_PORT = 1389; @Container private static final GenericContainer LDAP_CONTAINER = new GenericContainer<>(DockerImage.LDAP_IMAGE) diff --git a/datavault-common/src/test/java/org/datavaultplatform/common/task/TaskExecutorTest.java b/datavault-common/src/test/java/org/datavaultplatform/common/task/TaskExecutorTest.java index 6d6e5403a..be3fd5112 100644 --- a/datavault-common/src/test/java/org/datavaultplatform/common/task/TaskExecutorTest.java +++ b/datavault-common/src/test/java/org/datavaultplatform/common/task/TaskExecutorTest.java @@ -7,11 +7,13 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.TimeUnit; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.*; -public class TaskExecutorTest { +class TaskExecutorTest { @Test void testSingleExecution() throws Exception { @@ -56,6 +58,28 @@ void testTaskExecutionException() { assertEquals("bob123", ex.getMessage()); } + @Test + void testTasksSubmittedBeforeErrorTaskCanRunToCompletion() { + List finishedOkay = new CopyOnWriteArrayList<>(); + TaskExecutor executor = new TaskExecutor<>(1, "Error"); + for (int taskNum = 1; taskNum <= 3; taskNum++) { + final String taskNumString = String.valueOf(taskNum); + executor.add(() -> { + String msg = "finishedOkay-" + taskNumString; + finishedOkay.add(msg); + return msg; + }); + } + executor.add(() -> { + throw new IOException("oops!"); + }); + IOException ex = assertThrows(IOException.class, () -> { + executor.execute(System.out::println); + }); + assertThat(ex).hasMessage("oops!"); + assertThat(finishedOkay).isEqualTo(List.of("finishedOkay-1","finishedOkay-2","finishedOkay-3")); + } + @Test void testNullTasksRejected() { TaskExecutor executor = new TaskExecutor<>(1, "Error"); @@ -92,9 +116,13 @@ void testParallelExecution() throws Exception { } private Callable getDelayedTask(int delaySecs, T result) { + return getDelayedTask(delaySecs, () -> result); + } + + private Callable getDelayedTask(int delaySecs, Callable result) { return () -> { TimeUnit.SECONDS.sleep(delaySecs); - return result; + return result.call(); }; } diff --git a/datavault-webapp/src/main/webapp/WEB-INF/templates/layout/footer.html b/datavault-webapp/src/main/webapp/WEB-INF/templates/layout/footer.html index abc14357e..20cee4002 100644 --- a/datavault-webapp/src/main/webapp/WEB-INF/templates/layout/footer.html +++ b/datavault-webapp/src/main/webapp/WEB-INF/templates/layout/footer.html @@ -10,7 +10,7 @@

The University of Edinburgh

-
+
@@ -53,14 +53,13 @@

The University of Edinburgh

which has been granted degree awarding powers.

-
-
+

Unless explicitly stated otherwise, all material is copyright © The University of Edinburgh 2026.

@@ -69,7 +68,7 @@

The University of Edinburgh